diff --git a/lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/INSTALLER b/lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/METADATA b/lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..03a6f0f7ff91628f60c4a95c4f3acbfa8d654ea8 --- /dev/null +++ b/lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/METADATA @@ -0,0 +1,112 @@ +Metadata-Version: 2.4 +Name: aiosignal +Version: 1.4.0 +Summary: aiosignal: a list of registered asynchronous callbacks +Home-page: https://github.com/aio-libs/aiosignal +Maintainer: aiohttp team +Maintainer-email: team@aiohttp.org +License: Apache 2.0 +Project-URL: Chat: Gitter, https://gitter.im/aio-libs/Lobby +Project-URL: CI: GitHub Actions, https://github.com/aio-libs/aiosignal/actions +Project-URL: Coverage: codecov, https://codecov.io/github/aio-libs/aiosignal +Project-URL: Docs: RTD, https://docs.aiosignal.org +Project-URL: GitHub: issues, https://github.com/aio-libs/aiosignal/issues +Project-URL: GitHub: repo, https://github.com/aio-libs/aiosignal +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Intended Audience :: Developers +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Development Status :: 5 - Production/Stable +Classifier: Operating System :: POSIX +Classifier: Operating System :: MacOS :: MacOS X +Classifier: Operating System :: Microsoft :: Windows +Classifier: Framework :: AsyncIO +Requires-Python: >=3.9 +Description-Content-Type: text/x-rst +License-File: LICENSE +Requires-Dist: frozenlist>=1.1.0 +Requires-Dist: typing-extensions>=4.2; python_version < "3.13" +Dynamic: license-file + +========= +aiosignal +========= + +.. image:: https://github.com/aio-libs/aiosignal/workflows/CI/badge.svg + :target: https://github.com/aio-libs/aiosignal/actions?query=workflow%3ACI + :alt: GitHub status for master branch + +.. image:: https://codecov.io/gh/aio-libs/aiosignal/branch/master/graph/badge.svg?flag=pytest + :target: https://codecov.io/gh/aio-libs/aiosignal?flags[0]=pytest + :alt: codecov.io status for master branch + +.. image:: https://badge.fury.io/py/aiosignal.svg + :target: https://pypi.org/project/aiosignal + :alt: Latest PyPI package version + +.. image:: https://readthedocs.org/projects/aiosignal/badge/?version=latest + :target: https://aiosignal.readthedocs.io/ + :alt: Latest Read The Docs + +.. image:: https://img.shields.io/discourse/topics?server=https%3A%2F%2Faio-libs.discourse.group%2F + :target: https://aio-libs.discourse.group/ + :alt: Discourse group for io-libs + +.. image:: https://badges.gitter.im/Join%20Chat.svg + :target: https://gitter.im/aio-libs/Lobby + :alt: Chat on Gitter + +Introduction +============ + +A project to manage callbacks in `asyncio` projects. + +``Signal`` is a list of registered asynchronous callbacks. + +The signal's life-cycle has two stages: after creation its content +could be filled by using standard list operations: ``sig.append()`` +etc. + +After you call ``sig.freeze()`` the signal is *frozen*: adding, removing +and dropping callbacks is forbidden. + +The only available operation is calling the previously registered +callbacks by using ``await sig.send(data)``. + +For concrete usage examples see the `Signals + +section of the `Web Server Advanced +` chapter of the `aiohttp +documentation`_. + + +Installation +------------ + +:: + + $ pip install aiosignal + + +Documentation +============= + +https://aiosignal.readthedocs.io/ + +License +======= + +``aiosignal`` is offered under the Apache 2 license. + +Source code +=========== + +The project is hosted on GitHub_ + +Please file an issue in the `bug tracker +`_ if you have found a bug +or have some suggestions to improve the library. + +.. _GitHub: https://github.com/aio-libs/aiosignal +.. _aiohttp documentation: https://docs.aiohttp.org/ diff --git a/lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/RECORD b/lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..df5ca06f6fe1bbec1a02456b5b001a34771ecced --- /dev/null +++ b/lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/RECORD @@ -0,0 +1,9 @@ +aiosignal-1.4.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +aiosignal-1.4.0.dist-info/METADATA,sha256=CSR-8dqLxpZyjUcTDnAuQwf299EB1sSFv_nzpxznAI0,3662 +aiosignal-1.4.0.dist-info/RECORD,, +aiosignal-1.4.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91 +aiosignal-1.4.0.dist-info/licenses/LICENSE,sha256=b9UkPpLdf5jsacesN3co50kFcJ_1J6W_mNbQJjwE9bY,11332 +aiosignal-1.4.0.dist-info/top_level.txt,sha256=z45aNOKGDdrI1roqZY3BGXQ22kJFPHBmVdwtLYLtXC0,10 +aiosignal/__init__.py,sha256=TIkmUG9HTBt4dfq2nISYBiZiRB2xwvFtEZydLP0HPL4,1537 +aiosignal/__pycache__/__init__.cpython-312.pyc,, +aiosignal/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 diff --git a/lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/WHEEL b/lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..e7fa31b6f3f78deb1022c1f7927f07d4d16da822 --- /dev/null +++ b/lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: setuptools (80.9.0) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/licenses/LICENSE b/lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..7082a2d5b9047bfc09589f387053e24ea490bc54 --- /dev/null +++ b/lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/licenses/LICENSE @@ -0,0 +1,201 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2013-2019 Nikolay Kim and Andrew Svetlov + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/top_level.txt b/lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..ac6df3afe74a5fd43afc7ab7f8393571a495fdc5 --- /dev/null +++ b/lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/top_level.txt @@ -0,0 +1 @@ +aiosignal diff --git a/lib/python3.12/site-packages/datasets/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..211489dcfe0570a63125edeaa72756388b8672d2 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/__pycache__/arrow_reader.cpython-312.pyc b/lib/python3.12/site-packages/datasets/__pycache__/arrow_reader.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3e90d283ed4e52bfadeaf3448bc427b88fd4f2b0 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/__pycache__/arrow_reader.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/__pycache__/arrow_writer.cpython-312.pyc b/lib/python3.12/site-packages/datasets/__pycache__/arrow_writer.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e6c69728a1ba599a2c4b05c6b81988f1b32bc3f4 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/__pycache__/arrow_writer.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/__pycache__/builder.cpython-312.pyc b/lib/python3.12/site-packages/datasets/__pycache__/builder.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6b774efce4c783a1bdf74b22a73a2d7336a3aa60 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/__pycache__/builder.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/__pycache__/combine.cpython-312.pyc b/lib/python3.12/site-packages/datasets/__pycache__/combine.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b4f803d811aa37d6c6eb1644e2e203a2a5c25590 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/__pycache__/combine.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/__pycache__/config.cpython-312.pyc b/lib/python3.12/site-packages/datasets/__pycache__/config.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a5c9f3aee8fa307010ead3b589e5bc09a5a4e56c Binary files /dev/null and b/lib/python3.12/site-packages/datasets/__pycache__/config.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/__pycache__/data_files.cpython-312.pyc b/lib/python3.12/site-packages/datasets/__pycache__/data_files.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..02573a0d6a65499fe693947023c3133c2f09263c Binary files /dev/null and b/lib/python3.12/site-packages/datasets/__pycache__/data_files.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/__pycache__/distributed.cpython-312.pyc b/lib/python3.12/site-packages/datasets/__pycache__/distributed.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f7754f1643ad10a360c4d5ac7879ba866dbeea95 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/__pycache__/distributed.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/__pycache__/exceptions.cpython-312.pyc b/lib/python3.12/site-packages/datasets/__pycache__/exceptions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..39ffce8bbfae2cc5b41bd71481a4d792c57211d8 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/__pycache__/exceptions.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/__pycache__/fingerprint.cpython-312.pyc b/lib/python3.12/site-packages/datasets/__pycache__/fingerprint.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f5d36a23f9d9c1bc934b3f688b7601b30f7cd3d8 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/__pycache__/fingerprint.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/__pycache__/hub.cpython-312.pyc b/lib/python3.12/site-packages/datasets/__pycache__/hub.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7a0baa17626ba0ed0d56e432cf9990492c8ec61b Binary files /dev/null and b/lib/python3.12/site-packages/datasets/__pycache__/hub.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/__pycache__/info.cpython-312.pyc b/lib/python3.12/site-packages/datasets/__pycache__/info.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d4804f1aa618854d7027c8843585347d5eeb8cc5 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/__pycache__/info.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/__pycache__/inspect.cpython-312.pyc b/lib/python3.12/site-packages/datasets/__pycache__/inspect.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..52fb83f92805a78628ad9667c089c3742daf277d Binary files /dev/null and b/lib/python3.12/site-packages/datasets/__pycache__/inspect.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/__pycache__/keyhash.cpython-312.pyc b/lib/python3.12/site-packages/datasets/__pycache__/keyhash.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6bd9a38d636e3708ab2258be5a4178ece46677be Binary files /dev/null and b/lib/python3.12/site-packages/datasets/__pycache__/keyhash.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/__pycache__/load.cpython-312.pyc b/lib/python3.12/site-packages/datasets/__pycache__/load.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e9672685dcead29a60e8e3dc4b1c1d4c6f75d257 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/__pycache__/load.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/__pycache__/naming.cpython-312.pyc b/lib/python3.12/site-packages/datasets/__pycache__/naming.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..87da48dc9021a540da51ddee4d252c6d2a05a979 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/__pycache__/naming.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/__pycache__/search.cpython-312.pyc b/lib/python3.12/site-packages/datasets/__pycache__/search.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b7cf233b53f1564925548cc353dcc2b7f51754d8 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/__pycache__/search.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/__pycache__/splits.cpython-312.pyc b/lib/python3.12/site-packages/datasets/__pycache__/splits.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ec3caee434b9788b861f19f4e61ccc847a506020 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/__pycache__/splits.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/__pycache__/streaming.cpython-312.pyc b/lib/python3.12/site-packages/datasets/__pycache__/streaming.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2aca951fffc169274357579ffa24f9104554902c Binary files /dev/null and b/lib/python3.12/site-packages/datasets/__pycache__/streaming.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/download/__init__.py b/lib/python3.12/site-packages/datasets/download/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8ae0d436504dc2e609b0ca8851509c72a161dbde --- /dev/null +++ b/lib/python3.12/site-packages/datasets/download/__init__.py @@ -0,0 +1,10 @@ +__all__ = [ + "DownloadConfig", + "DownloadManager", + "DownloadMode", + "StreamingDownloadManager", +] + +from .download_config import DownloadConfig +from .download_manager import DownloadManager, DownloadMode +from .streaming_download_manager import StreamingDownloadManager diff --git a/lib/python3.12/site-packages/datasets/download/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/download/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3f8b51ad3c0459dd095b64ceebe54703bf691ade Binary files /dev/null and b/lib/python3.12/site-packages/datasets/download/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/download/__pycache__/download_config.cpython-312.pyc b/lib/python3.12/site-packages/datasets/download/__pycache__/download_config.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4acd93a731673ceeecbac18be17ace0f98411ef8 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/download/__pycache__/download_config.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/download/__pycache__/download_manager.cpython-312.pyc b/lib/python3.12/site-packages/datasets/download/__pycache__/download_manager.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f0410f59b29bdae912cfb2a4d1e37ff3f7baf503 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/download/__pycache__/download_manager.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/download/__pycache__/streaming_download_manager.cpython-312.pyc b/lib/python3.12/site-packages/datasets/download/__pycache__/streaming_download_manager.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5b30401129999de722620f68fd9d421a90ed41e6 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/download/__pycache__/streaming_download_manager.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/download/download_config.py b/lib/python3.12/site-packages/datasets/download/download_config.py new file mode 100644 index 0000000000000000000000000000000000000000..b9e539fb0536105c3bc04680aca81b7d5bdd4e15 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/download/download_config.py @@ -0,0 +1,81 @@ +import copy +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Optional, Union + +from .. import config + + +@dataclass +class DownloadConfig: + """Configuration for our cached path manager. + + Attributes: + cache_dir (`str` or `Path`, *optional*): + Specify a cache directory to save the file to (overwrite the + default cache dir). + force_download (`bool`, defaults to `False`): + If `True`, re-download the file even if it's already cached in + the cache dir. + resume_download (`bool`, defaults to `False`): + If `True`, resume the download if an incompletely received file is + found. + proxies (`dict`, *optional*): + user_agent (`str`, *optional*): + Optional string or dict that will be appended to the user-agent on remote + requests. + extract_compressed_file (`bool`, defaults to `False`): + If `True` and the path point to a zip or tar file, + extract the compressed file in a folder along the archive. + force_extract (`bool`, defaults to `False`): + If `True` when `extract_compressed_file` is `True` and the archive + was already extracted, re-extract the archive and override the folder where it was extracted. + delete_extracted (`bool`, defaults to `False`): + Whether to delete (or keep) the extracted files. + extract_on_the_fly (`bool`, defaults to `False`): + If `True`, extract compressed files while they are being read. + use_etag (`bool`, defaults to `True`): + Whether to use the ETag HTTP response header to validate the cached files. + num_proc (`int`, *optional*): + The number of processes to launch to download the files in parallel. + max_retries (`int`, default to `1`): + The number of times to retry an HTTP request if it fails. + token (`str` or `bool`, *optional*): + Optional string or boolean to use as Bearer token + for remote files on the Datasets Hub. If `True`, or not specified, will get token from `~/.huggingface`. + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the dataset file-system backend, if any. + download_desc (`str`, *optional*): + A description to be displayed alongside with the progress bar while downloading the files. + disable_tqdm (`bool`, defaults to `False`): + Whether to disable the individual files download progress bar + """ + + cache_dir: Optional[Union[str, Path]] = None + force_download: bool = False + resume_download: bool = False + local_files_only: bool = False + proxies: Optional[dict] = None + user_agent: Optional[str] = None + extract_compressed_file: bool = False + force_extract: bool = False + delete_extracted: bool = False + extract_on_the_fly: bool = False + use_etag: bool = True + num_proc: Optional[int] = None + max_retries: int = 1 + token: Optional[Union[str, bool]] = None + storage_options: dict[str, Any] = field(default_factory=dict) + download_desc: Optional[str] = None + disable_tqdm: bool = False + + def copy(self) -> "DownloadConfig": + return self.__class__(**{k: copy.deepcopy(v) for k, v in self.__dict__.items()}) + + def __setattr__(self, name, value): + if name == "token" and getattr(self, "storage_options", None) is not None: + if "hf" not in self.storage_options: + self.storage_options["hf"] = {"token": value, "endpoint": config.HF_ENDPOINT} + elif getattr(self.storage_options["hf"], "token", None) is None: + self.storage_options["hf"]["token"] = value + super().__setattr__(name, value) diff --git a/lib/python3.12/site-packages/datasets/download/download_manager.py b/lib/python3.12/site-packages/datasets/download/download_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..b6ee1d28e2b3dd203d62d27137861b33651324db --- /dev/null +++ b/lib/python3.12/site-packages/datasets/download/download_manager.py @@ -0,0 +1,340 @@ +# Copyright 2020 The TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Download manager interface.""" + +import enum +import io +import multiprocessing +import os +from datetime import datetime +from functools import partial +from typing import Optional, Union + +import fsspec +from fsspec.core import url_to_fs +from tqdm.contrib.concurrent import thread_map + +from .. import config +from ..utils import tqdm as hf_tqdm +from ..utils.file_utils import ( + ArchiveIterable, + FilesIterable, + cached_path, + is_relative_path, + stack_multiprocessing_download_progress_bars, + url_or_path_join, +) +from ..utils.info_utils import get_size_checksum_dict +from ..utils.logging import get_logger, tqdm +from ..utils.py_utils import NestedDataStructure, map_nested +from ..utils.track import tracked_str +from .download_config import DownloadConfig + + +logger = get_logger(__name__) + + +class DownloadMode(enum.Enum): + """`Enum` for how to treat pre-existing downloads and data. + + The default mode is `REUSE_DATASET_IF_EXISTS`, which will reuse both + raw downloads and the prepared dataset if they exist. + + The generations modes: + + | | Downloads | Dataset | + |-------------------------------------|-----------|---------| + | `REUSE_DATASET_IF_EXISTS` (default) | Reuse | Reuse | + | `REUSE_CACHE_IF_EXISTS` | Reuse | Fresh | + | `FORCE_REDOWNLOAD` | Fresh | Fresh | + + """ + + REUSE_DATASET_IF_EXISTS = "reuse_dataset_if_exists" + REUSE_CACHE_IF_EXISTS = "reuse_cache_if_exists" + FORCE_REDOWNLOAD = "force_redownload" + + +class DownloadManager: + is_streaming = False + + def __init__( + self, + dataset_name: Optional[str] = None, + data_dir: Optional[str] = None, + download_config: Optional[DownloadConfig] = None, + base_path: Optional[str] = None, + record_checksums=True, + ): + """Download manager constructor. + + Args: + data_dir: + can be used to specify a manual directory to get the files from. + dataset_name (`str`): + name of dataset this instance will be used for. If + provided, downloads will contain which datasets they were used for. + download_config (`DownloadConfig`): + to specify the cache directory and other + download options + base_path (`str`): + base path that is used when relative paths are used to + download files. This can be a remote url. + record_checksums (`bool`, defaults to `True`): + Whether to record the checksums of the downloaded files. If None, the value is inferred from the builder. + """ + self._dataset_name = dataset_name + self._data_dir = data_dir + self._base_path = base_path or os.path.abspath(".") + # To record what is being used: {url: {num_bytes: int, checksum: str}} + self._recorded_sizes_checksums: dict[str, dict[str, Optional[Union[int, str]]]] = {} + self.record_checksums = record_checksums + self.download_config = download_config or DownloadConfig() + self.downloaded_paths = {} + self.extracted_paths = {} + + @property + def manual_dir(self): + return self._data_dir + + @property + def downloaded_size(self): + """Returns the total size of downloaded files.""" + return sum(checksums_dict["num_bytes"] for checksums_dict in self._recorded_sizes_checksums.values()) + + def _record_sizes_checksums(self, url_or_urls: NestedDataStructure, downloaded_path_or_paths: NestedDataStructure): + """Record size/checksum of downloaded files.""" + delay = 5 + for url, path in hf_tqdm( + list(zip(url_or_urls.flatten(), downloaded_path_or_paths.flatten())), + delay=delay, + desc="Computing checksums", + ): + # call str to support PathLike objects + self._recorded_sizes_checksums[str(url)] = get_size_checksum_dict( + path, record_checksum=self.record_checksums + ) + + def download(self, url_or_urls): + """Download given URL(s). + + By default, only one process is used for download. Pass customized `download_config.num_proc` to change this behavior. + + Args: + url_or_urls (`str` or `list` or `dict`): + URL or `list` or `dict` of URLs to download. Each URL is a `str`. + + Returns: + `str` or `list` or `dict`: + The downloaded paths matching the given input `url_or_urls`. + + Example: + + ```py + >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') + ``` + """ + download_config = self.download_config.copy() + download_config.extract_compressed_file = False + if download_config.download_desc is None: + download_config.download_desc = "Downloading data" + + download_func = partial(self._download_batched, download_config=download_config) + + start_time = datetime.now() + with stack_multiprocessing_download_progress_bars(): + downloaded_path_or_paths = map_nested( + download_func, + url_or_urls, + map_tuple=True, + num_proc=download_config.num_proc, + desc="Downloading data files", + batched=True, + batch_size=-1, + ) + duration = datetime.now() - start_time + logger.info(f"Downloading took {duration.total_seconds() // 60} min") + url_or_urls = NestedDataStructure(url_or_urls) + downloaded_path_or_paths = NestedDataStructure(downloaded_path_or_paths) + self.downloaded_paths.update(dict(zip(url_or_urls.flatten(), downloaded_path_or_paths.flatten()))) + + start_time = datetime.now() + self._record_sizes_checksums(url_or_urls, downloaded_path_or_paths) + duration = datetime.now() - start_time + logger.info(f"Checksum Computation took {duration.total_seconds() // 60} min") + + return downloaded_path_or_paths.data + + def _download_batched( + self, + url_or_filenames: list[str], + download_config: DownloadConfig, + ) -> list[str]: + if len(url_or_filenames) >= 16: + download_config = download_config.copy() + download_config.disable_tqdm = True + download_func = partial(self._download_single, download_config=download_config) + + fs: fsspec.AbstractFileSystem + path = str(url_or_filenames[0]) + if is_relative_path(path): + # append the relative path to the base_path + path = url_or_path_join(self._base_path, path) + fs, path = url_to_fs(path, **download_config.storage_options) + size = 0 + try: + size = fs.info(path).get("size", 0) + except Exception: + pass + max_workers = ( + config.HF_DATASETS_MULTITHREADING_MAX_WORKERS if size < (20 << 20) else 1 + ) # enable multithreading if files are small + + return thread_map( + download_func, + url_or_filenames, + desc=download_config.download_desc or "Downloading", + unit="files", + position=multiprocessing.current_process()._identity[-1] # contains the ranks of subprocesses + if os.environ.get("HF_DATASETS_STACK_MULTIPROCESSING_DOWNLOAD_PROGRESS_BARS") == "1" + and multiprocessing.current_process()._identity + else None, + max_workers=max_workers, + tqdm_class=tqdm, + ) + else: + return [ + self._download_single(url_or_filename, download_config=download_config) + for url_or_filename in url_or_filenames + ] + + def _download_single(self, url_or_filename: str, download_config: DownloadConfig) -> str: + url_or_filename = str(url_or_filename) + if is_relative_path(url_or_filename): + # append the relative path to the base_path + url_or_filename = url_or_path_join(self._base_path, url_or_filename) + out = cached_path(url_or_filename, download_config=download_config) + out = tracked_str(out) + out.set_origin(url_or_filename) + return out + + def iter_archive(self, path_or_buf: Union[str, io.BufferedReader]): + """Iterate over files within an archive. + + Args: + path_or_buf (`str` or `io.BufferedReader`): + Archive path or archive binary file object. + + Yields: + `tuple[str, io.BufferedReader]`: + 2-tuple (path_within_archive, file_object). + File object is opened in binary mode. + + Example: + + ```py + >>> archive = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') + >>> files = dl_manager.iter_archive(archive) + ``` + """ + + if hasattr(path_or_buf, "read"): + return ArchiveIterable.from_buf(path_or_buf) + else: + return ArchiveIterable.from_urlpath(path_or_buf) + + def iter_files(self, paths: Union[str, list[str]]): + """Iterate over file paths. + + Args: + paths (`str` or `list` of `str`): + Root paths. + + Yields: + `str`: File path. + + Example: + + ```py + >>> files = dl_manager.download_and_extract('https://huggingface.co/datasets/beans/resolve/main/data/train.zip') + >>> files = dl_manager.iter_files(files) + ``` + """ + return FilesIterable.from_urlpaths(paths) + + def extract(self, path_or_paths): + """Extract given path(s). + + Args: + path_or_paths (path or `list` or `dict`): + Path of file to extract. Each path is a `str`. + + Returns: + extracted_path(s): `str`, The extracted paths matching the given input + path_or_paths. + + Example: + + ```py + >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') + >>> extracted_files = dl_manager.extract(downloaded_files) + ``` + """ + download_config = self.download_config.copy() + download_config.extract_compressed_file = True + extract_func = partial(self._download_single, download_config=download_config) + extracted_paths = map_nested( + extract_func, + path_or_paths, + num_proc=download_config.num_proc, + desc="Extracting data files", + ) + path_or_paths = NestedDataStructure(path_or_paths) + extracted_paths = NestedDataStructure(extracted_paths) + self.extracted_paths.update(dict(zip(path_or_paths.flatten(), extracted_paths.flatten()))) + return extracted_paths.data + + def download_and_extract(self, url_or_urls): + """Download and extract given `url_or_urls`. + + Is roughly equivalent to: + + ``` + extracted_paths = dl_manager.extract(dl_manager.download(url_or_urls)) + ``` + + Args: + url_or_urls (`str` or `list` or `dict`): + URL or `list` or `dict` of URLs to download and extract. Each URL is a `str`. + + Returns: + extracted_path(s): `str`, extracted paths of given URL(s). + """ + return self.extract(self.download(url_or_urls)) + + def get_recorded_sizes_checksums(self): + return self._recorded_sizes_checksums.copy() + + def delete_extracted_files(self): + paths_to_delete = set(self.extracted_paths.values()) - set(self.downloaded_paths.values()) + for key, path in list(self.extracted_paths.items()): + if path in paths_to_delete and os.path.isfile(path): + os.remove(path) + del self.extracted_paths[key] + + def manage_extracted_files(self): + if self.download_config.delete_extracted: + self.delete_extracted_files() diff --git a/lib/python3.12/site-packages/datasets/download/streaming_download_manager.py b/lib/python3.12/site-packages/datasets/download/streaming_download_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..ff2dc1a64bd95616ca78f515e0ee2f14becfc5f6 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/download/streaming_download_manager.py @@ -0,0 +1,219 @@ +import io +import os +from collections.abc import Iterable +from typing import Optional, Union + +from ..utils.file_utils import ( # noqa: F401 # backward compatibility + SINGLE_FILE_COMPRESSION_PROTOCOLS, + ArchiveIterable, + FilesIterable, + _get_extraction_protocol, + _get_path_extension, + _prepare_path_and_storage_options, + is_relative_path, + url_or_path_join, + xbasename, + xdirname, + xet_parse, + xexists, + xgetsize, + xglob, + xgzip_open, + xisdir, + xisfile, + xjoin, + xlistdir, + xnumpy_load, + xopen, + xpandas_read_csv, + xpandas_read_excel, + xPath, + xpyarrow_parquet_read_table, + xrelpath, + xsio_loadmat, + xsplit, + xsplitext, + xwalk, + xxml_dom_minidom_parse, +) +from ..utils.logging import get_logger +from ..utils.py_utils import map_nested +from .download_config import DownloadConfig + + +logger = get_logger(__name__) + + +class StreamingDownloadManager: + """ + Download manager that uses the "::" separator to navigate through (possibly remote) compressed archives. + Contrary to the regular `DownloadManager`, the `download` and `extract` methods don't actually download nor extract + data, but they rather return the path or url that could be opened using the `xopen` function which extends the + built-in `open` function to stream data from remote files. + """ + + is_streaming = True + + def __init__( + self, + dataset_name: Optional[str] = None, + data_dir: Optional[str] = None, + download_config: Optional[DownloadConfig] = None, + base_path: Optional[str] = None, + ): + self._dataset_name = dataset_name + self._data_dir = data_dir + self._base_path = base_path or os.path.abspath(".") + self.download_config = download_config or DownloadConfig() + self.downloaded_size = None + self.record_checksums = False + + @property + def manual_dir(self): + return self._data_dir + + def download(self, url_or_urls): + """Normalize URL(s) of files to stream data from. + This is the lazy version of `DownloadManager.download` for streaming. + + Args: + url_or_urls (`str` or `list` or `dict`): + URL(s) of files to stream data from. Each url is a `str`. + + Returns: + url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input url_or_urls. + + Example: + + ```py + >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') + ``` + """ + url_or_urls = map_nested(self._download_single, url_or_urls, map_tuple=True) + return url_or_urls + + def _download_single(self, urlpath: str) -> str: + urlpath = str(urlpath) + if is_relative_path(urlpath): + # append the relative path to the base_path + urlpath = url_or_path_join(self._base_path, urlpath) + return urlpath + + def extract(self, url_or_urls): + """Add extraction protocol for given url(s) for streaming. + + This is the lazy version of `DownloadManager.extract` for streaming. + + Args: + url_or_urls (`str` or `list` or `dict`): + URL(s) of files to stream data from. Each url is a `str`. + + Returns: + url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input `url_or_urls`. + + Example: + + ```py + >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') + >>> extracted_files = dl_manager.extract(downloaded_files) + ``` + """ + urlpaths = map_nested(self._extract, url_or_urls, map_tuple=True) + return urlpaths + + def _extract(self, urlpath: str) -> str: + urlpath = str(urlpath) + protocol = _get_extraction_protocol(urlpath, download_config=self.download_config) + # get inner file: zip://train-00000.json.gz::https://foo.bar/data.zip -> zip://train-00000.json.gz + path = urlpath.split("::")[0] + extension = _get_path_extension(path) + if extension in ["tgz", "tar"] or path.endswith((".tar.gz", ".tar.bz2", ".tar.xz")): + raise NotImplementedError( + f"Extraction protocol for TAR archives like '{urlpath}' is not implemented in streaming mode. " + f"Please use `dl_manager.iter_archive` instead.\n\n" + f"Example usage:\n\n" + f"\turl = dl_manager.download(url)\n" + f"\ttar_archive_iterator = dl_manager.iter_archive(url)\n\n" + f"\tfor filename, file in tar_archive_iterator:\n" + f"\t\t..." + ) + if protocol is None: + # no extraction + return urlpath + elif protocol in SINGLE_FILE_COMPRESSION_PROTOCOLS: + # there is one single file which is the uncompressed file + inner_file = os.path.basename(urlpath.split("::")[0]) + inner_file = inner_file[: inner_file.rindex(".")] if "." in inner_file else inner_file + return f"{protocol}://{inner_file}::{urlpath}" + else: + return f"{protocol}://::{urlpath}" + + def download_and_extract(self, url_or_urls): + """Prepare given `url_or_urls` for streaming (add extraction protocol). + + This is the lazy version of `DownloadManager.download_and_extract` for streaming. + + Is equivalent to: + + ``` + urls = dl_manager.extract(dl_manager.download(url_or_urls)) + ``` + + Args: + url_or_urls (`str` or `list` or `dict`): + URL(s) to stream from data from. Each url is a `str`. + + Returns: + url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input `url_or_urls`. + """ + return self.extract(self.download(url_or_urls)) + + def iter_archive(self, urlpath_or_buf: Union[str, io.BufferedReader]) -> Iterable[tuple]: + """Iterate over files within an archive. + + Args: + urlpath_or_buf (`str` or `io.BufferedReader`): + Archive path or archive binary file object. + + Yields: + `tuple[str, io.BufferedReader]`: + 2-tuple (path_within_archive, file_object). + File object is opened in binary mode. + + Example: + + ```py + >>> archive = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') + >>> files = dl_manager.iter_archive(archive) + ``` + """ + + if hasattr(urlpath_or_buf, "read"): + return ArchiveIterable.from_buf(urlpath_or_buf) + else: + return ArchiveIterable.from_urlpath(urlpath_or_buf, download_config=self.download_config) + + def iter_files(self, urlpaths: Union[str, list[str]]) -> Iterable[str]: + """Iterate over files. + + Args: + urlpaths (`str` or `list` of `str`): + Root paths. + + Yields: + str: File URL path. + + Example: + + ```py + >>> files = dl_manager.download_and_extract('https://huggingface.co/datasets/beans/resolve/main/data/train.zip') + >>> files = dl_manager.iter_files(files) + ``` + """ + return FilesIterable.from_urlpaths(urlpaths, download_config=self.download_config) + + def manage_extracted_files(self): + pass + + def get_recorded_sizes_checksums(self): + pass diff --git a/lib/python3.12/site-packages/datasets/features/__init__.py b/lib/python3.12/site-packages/datasets/features/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..95bb1cf108066e702965191fbf781a0b47e7fb55 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/features/__init__.py @@ -0,0 +1,23 @@ +__all__ = [ + "Audio", + "Array2D", + "Array3D", + "Array4D", + "Array5D", + "ClassLabel", + "Features", + "LargeList", + "Sequence", + "Value", + "Image", + "Translation", + "TranslationVariableLanguages", + "Video", + "Pdf", +] +from .audio import Audio +from .features import Array2D, Array3D, Array4D, Array5D, ClassLabel, Features, LargeList, Sequence, Value +from .image import Image +from .pdf import Pdf +from .translation import Translation, TranslationVariableLanguages +from .video import Video diff --git a/lib/python3.12/site-packages/datasets/features/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/features/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3988f5bd23e378092a400b0e294e55b3ecaaaaa8 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/features/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/features/__pycache__/audio.cpython-312.pyc b/lib/python3.12/site-packages/datasets/features/__pycache__/audio.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e12b7131f08ea4136c727b0b167b3389e7471f65 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/features/__pycache__/audio.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/features/__pycache__/image.cpython-312.pyc b/lib/python3.12/site-packages/datasets/features/__pycache__/image.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ef27c854556c7bd10cc94f88e51ec5fee30596ac Binary files /dev/null and b/lib/python3.12/site-packages/datasets/features/__pycache__/image.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/features/__pycache__/pdf.cpython-312.pyc b/lib/python3.12/site-packages/datasets/features/__pycache__/pdf.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..26208bf3568d34aa1787fb0c50ae913b43e810a0 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/features/__pycache__/pdf.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/features/__pycache__/translation.cpython-312.pyc b/lib/python3.12/site-packages/datasets/features/__pycache__/translation.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8a6d4596a6986c327a8219dbe9a6013f215bed61 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/features/__pycache__/translation.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/features/__pycache__/video.cpython-312.pyc b/lib/python3.12/site-packages/datasets/features/__pycache__/video.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d1a88dcb5c309ea04030a4337d64b6dbb4ea7857 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/features/__pycache__/video.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/features/audio.py b/lib/python3.12/site-packages/datasets/features/audio.py new file mode 100644 index 0000000000000000000000000000000000000000..82c2e7f1f469897f85ad97a59c7f5a83bbefd3e3 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/features/audio.py @@ -0,0 +1,274 @@ +import os +from dataclasses import dataclass, field +from io import BytesIO +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union + +import numpy as np +import pyarrow as pa + +from .. import config +from ..download.download_config import DownloadConfig +from ..table import array_cast +from ..utils.file_utils import xopen, xsplitext +from ..utils.py_utils import no_op_if_value_is_null, string_to_dict + + +if TYPE_CHECKING: + from .features import FeatureType + + +@dataclass +class Audio: + """Audio [`Feature`] to extract audio data from an audio file. + + Input: The Audio feature accepts as input: + - A `str`: Absolute path to the audio file (i.e. random access is allowed). + - A `dict` with the keys: + + - `path`: String with relative path of the audio file to the archive file. + - `bytes`: Bytes content of the audio file. + + This is useful for archived files with sequential access. + + - A `dict` with the keys: + + - `path`: String with relative path of the audio file to the archive file. + - `array`: Array containing the audio sample + - `sampling_rate`: Integer corresponding to the sampling rate of the audio sample. + + This is useful for archived files with sequential access. + + Args: + sampling_rate (`int`, *optional*): + Target sampling rate. If `None`, the native sampling rate is used. + mono (`bool`, defaults to `True`): + Whether to convert the audio signal to mono by averaging samples across + channels. + decode (`bool`, defaults to `True`): + Whether to decode the audio data. If `False`, + returns the underlying dictionary in the format `{"path": audio_path, "bytes": audio_bytes}`. + + Example: + + ```py + >>> from datasets import load_dataset, Audio + >>> ds = load_dataset("PolyAI/minds14", name="en-US", split="train") + >>> ds = ds.cast_column("audio", Audio(sampling_rate=16000)) + >>> ds[0]["audio"] + {'array': array([ 2.3443763e-05, 2.1729663e-04, 2.2145823e-04, ..., + 3.8356509e-05, -7.3497440e-06, -2.1754686e-05], dtype=float32), + 'path': '/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav', + 'sampling_rate': 16000} + ``` + """ + + sampling_rate: Optional[int] = None + mono: bool = True + decode: bool = True + id: Optional[str] = None + # Automatically constructed + dtype: ClassVar[str] = "dict" + pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()}) + _type: str = field(default="Audio", init=False, repr=False) + + def __call__(self): + return self.pa_type + + def encode_example(self, value: Union[str, bytes, bytearray, dict]) -> dict: + """Encode example into a format for Arrow. + + Args: + value (`str` or `dict`): + Data passed as input to Audio feature. + + Returns: + `dict` + """ + try: + import soundfile as sf # soundfile is a dependency of librosa, needed to decode audio files. + except ImportError as err: + raise ImportError("To support encoding audio data, please install 'soundfile'.") from err + if isinstance(value, str): + return {"bytes": None, "path": value} + elif isinstance(value, (bytes, bytearray)): + return {"bytes": value, "path": None} + elif "array" in value: + # convert the audio array to wav bytes + buffer = BytesIO() + sf.write(buffer, value["array"], value["sampling_rate"], format="wav") + return {"bytes": buffer.getvalue(), "path": None} + elif value.get("path") is not None and os.path.isfile(value["path"]): + # we set "bytes": None to not duplicate the data if they're already available locally + if value["path"].endswith("pcm"): + # "PCM" only has raw audio bytes + if value.get("sampling_rate") is None: + # At least, If you want to convert "PCM-byte" to "WAV-byte", you have to know sampling rate + raise KeyError("To use PCM files, please specify a 'sampling_rate' in Audio object") + if value.get("bytes"): + # If we already had PCM-byte, we don`t have to make "read file, make bytes" (just use it!) + bytes_value = np.frombuffer(value["bytes"], dtype=np.int16).astype(np.float32) / 32767 + else: + bytes_value = np.memmap(value["path"], dtype="h", mode="r").astype(np.float32) / 32767 + + buffer = BytesIO(b"") + sf.write(buffer, bytes_value, value["sampling_rate"], format="wav") + return {"bytes": buffer.getvalue(), "path": None} + else: + return {"bytes": None, "path": value.get("path")} + elif value.get("bytes") is not None or value.get("path") is not None: + # store the audio bytes, and path is used to infer the audio format using the file extension + return {"bytes": value.get("bytes"), "path": value.get("path")} + else: + raise ValueError( + f"An audio sample should have one of 'path' or 'bytes' but they are missing or None in {value}." + ) + + def decode_example( + self, value: dict, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None + ) -> dict: + """Decode example audio file into audio data. + + Args: + value (`dict`): + A dictionary with keys: + + - `path`: String with relative audio file path. + - `bytes`: Bytes of the audio file. + token_per_repo_id (`dict`, *optional*): + To access and decode + audio files from private repositories on the Hub, you can pass + a dictionary repo_id (`str`) -> token (`bool` or `str`) + + Returns: + `dict` + """ + if not self.decode: + raise RuntimeError("Decoding is disabled for this feature. Please use Audio(decode=True) instead.") + + path, file = (value["path"], BytesIO(value["bytes"])) if value["bytes"] is not None else (value["path"], None) + if path is None and file is None: + raise ValueError(f"An audio sample should have one of 'path' or 'bytes' but both are None in {value}.") + + try: + import librosa + import soundfile as sf + except ImportError as err: + raise ImportError("To support decoding audio files, please install 'librosa' and 'soundfile'.") from err + + audio_format = xsplitext(path)[1][1:].lower() if path is not None else None + if not config.IS_OPUS_SUPPORTED and audio_format == "opus": + raise RuntimeError( + "Decoding 'opus' files requires system library 'libsndfile'>=1.0.31, " + 'You can try to update `soundfile` python library: `pip install "soundfile>=0.12.1"`. ' + ) + elif not config.IS_MP3_SUPPORTED and audio_format == "mp3": + raise RuntimeError( + "Decoding 'mp3' files requires system library 'libsndfile'>=1.1.0, " + 'You can try to update `soundfile` python library: `pip install "soundfile>=0.12.1"`. ' + ) + + if file is None: + token_per_repo_id = token_per_repo_id or {} + source_url = path.split("::")[-1] + pattern = ( + config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL + ) + source_url_fields = string_to_dict(source_url, pattern) + token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None + + download_config = DownloadConfig(token=token) + with xopen(path, "rb", download_config=download_config) as f: + array, sampling_rate = sf.read(f) + + else: + array, sampling_rate = sf.read(file) + + array = array.T + if self.mono: + array = librosa.to_mono(array) + if self.sampling_rate and self.sampling_rate != sampling_rate: + array = librosa.resample(array, orig_sr=sampling_rate, target_sr=self.sampling_rate) + sampling_rate = self.sampling_rate + + return {"path": path, "array": array, "sampling_rate": sampling_rate} + + def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]: + """If in the decodable state, raise an error, otherwise flatten the feature into a dictionary.""" + from .features import Value + + if self.decode: + raise ValueError("Cannot flatten a decoded Audio feature.") + return { + "bytes": Value("binary"), + "path": Value("string"), + } + + def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray]) -> pa.StructArray: + """Cast an Arrow array to the Audio arrow storage type. + The Arrow types that can be converted to the Audio pyarrow storage type are: + + - `pa.string()` - it must contain the "path" data + - `pa.binary()` - it must contain the audio bytes + - `pa.struct({"bytes": pa.binary()})` + - `pa.struct({"path": pa.string()})` + - `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter + + Args: + storage (`Union[pa.StringArray, pa.StructArray]`): + PyArrow array to cast. + + Returns: + `pa.StructArray`: Array in the Audio arrow storage type, that is + `pa.struct({"bytes": pa.binary(), "path": pa.string()})` + """ + if pa.types.is_string(storage.type): + bytes_array = pa.array([None] * len(storage), type=pa.binary()) + storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_binary(storage.type): + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_struct(storage.type) and storage.type.get_all_field_indices("array"): + storage = pa.array([Audio().encode_example(x) if x is not None else None for x in storage.to_pylist()]) + elif pa.types.is_struct(storage.type): + if storage.type.get_field_index("bytes") >= 0: + bytes_array = storage.field("bytes") + else: + bytes_array = pa.array([None] * len(storage), type=pa.binary()) + if storage.type.get_field_index("path") >= 0: + path_array = storage.field("path") + else: + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null()) + return array_cast(storage, self.pa_type) + + def embed_storage(self, storage: pa.StructArray) -> pa.StructArray: + """Embed audio files into the Arrow array. + + Args: + storage (`pa.StructArray`): + PyArrow array to embed. + + Returns: + `pa.StructArray`: Array in the Audio arrow storage type, that is + `pa.struct({"bytes": pa.binary(), "path": pa.string()})`. + """ + + @no_op_if_value_is_null + def path_to_bytes(path): + with xopen(path, "rb") as f: + bytes_ = f.read() + return bytes_ + + bytes_array = pa.array( + [ + (path_to_bytes(x["path"]) if x["bytes"] is None else x["bytes"]) if x is not None else None + for x in storage.to_pylist() + ], + type=pa.binary(), + ) + path_array = pa.array( + [os.path.basename(path) if path is not None else None for path in storage.field("path").to_pylist()], + type=pa.string(), + ) + storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=bytes_array.is_null()) + return array_cast(storage, self.pa_type) diff --git a/lib/python3.12/site-packages/datasets/features/features.py b/lib/python3.12/site-packages/datasets/features/features.py new file mode 100644 index 0000000000000000000000000000000000000000..9d9b4974aa7a60d2a4eb3aa9c7416b6b612bb141 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/features/features.py @@ -0,0 +1,2332 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""This class handle features definition in datasets and some utilities to display table type.""" + +import copy +import json +import re +import sys +from collections.abc import Iterable, Mapping +from collections.abc import Sequence as SequenceABC +from collections.abc import Sequence as Sequence_ +from dataclasses import InitVar, dataclass, field, fields +from functools import reduce, wraps +from operator import mul +from typing import Any, Callable, ClassVar, Literal, Optional, Union + +import numpy as np +import pandas as pd +import pyarrow as pa +import pyarrow.compute as pc +import pyarrow.types +from pandas.api.extensions import ExtensionArray as PandasExtensionArray +from pandas.api.extensions import ExtensionDtype as PandasExtensionDtype + +from .. import config +from ..naming import camelcase_to_snakecase, snakecase_to_camelcase +from ..table import array_cast +from ..utils import experimental, logging +from ..utils.py_utils import asdict, first_non_null_value, zip_dict +from .audio import Audio +from .image import Image, encode_pil_image +from .pdf import Pdf, encode_pdfplumber_pdf +from .translation import Translation, TranslationVariableLanguages +from .video import Video + + +logger = logging.get_logger(__name__) + + +def _arrow_to_datasets_dtype(arrow_type: pa.DataType) -> str: + """ + _arrow_to_datasets_dtype takes a pyarrow.DataType and converts it to a datasets string dtype. + In effect, `dt == string_to_arrow(_arrow_to_datasets_dtype(dt))` + """ + if pyarrow.types.is_null(arrow_type): + return "null" + elif pyarrow.types.is_boolean(arrow_type): + return "bool" + elif pyarrow.types.is_int8(arrow_type): + return "int8" + elif pyarrow.types.is_int16(arrow_type): + return "int16" + elif pyarrow.types.is_int32(arrow_type): + return "int32" + elif pyarrow.types.is_int64(arrow_type): + return "int64" + elif pyarrow.types.is_uint8(arrow_type): + return "uint8" + elif pyarrow.types.is_uint16(arrow_type): + return "uint16" + elif pyarrow.types.is_uint32(arrow_type): + return "uint32" + elif pyarrow.types.is_uint64(arrow_type): + return "uint64" + elif pyarrow.types.is_float16(arrow_type): + return "float16" # pyarrow dtype is "halffloat" + elif pyarrow.types.is_float32(arrow_type): + return "float32" # pyarrow dtype is "float" + elif pyarrow.types.is_float64(arrow_type): + return "float64" # pyarrow dtype is "double" + elif pyarrow.types.is_time32(arrow_type): + return f"time32[{pa.type_for_alias(str(arrow_type)).unit}]" + elif pyarrow.types.is_time64(arrow_type): + return f"time64[{pa.type_for_alias(str(arrow_type)).unit}]" + elif pyarrow.types.is_timestamp(arrow_type): + if arrow_type.tz is None: + return f"timestamp[{arrow_type.unit}]" + elif arrow_type.tz: + return f"timestamp[{arrow_type.unit}, tz={arrow_type.tz}]" + else: + raise ValueError(f"Unexpected timestamp object {arrow_type}.") + elif pyarrow.types.is_date32(arrow_type): + return "date32" # pyarrow dtype is "date32[day]" + elif pyarrow.types.is_date64(arrow_type): + return "date64" # pyarrow dtype is "date64[ms]" + elif pyarrow.types.is_duration(arrow_type): + return f"duration[{arrow_type.unit}]" + elif pyarrow.types.is_decimal128(arrow_type): + return f"decimal128({arrow_type.precision}, {arrow_type.scale})" + elif pyarrow.types.is_decimal256(arrow_type): + return f"decimal256({arrow_type.precision}, {arrow_type.scale})" + elif pyarrow.types.is_binary(arrow_type): + return "binary" + elif pyarrow.types.is_large_binary(arrow_type): + return "large_binary" + elif pyarrow.types.is_string(arrow_type): + return "string" + elif pyarrow.types.is_large_string(arrow_type): + return "large_string" + elif pyarrow.types.is_dictionary(arrow_type): + return _arrow_to_datasets_dtype(arrow_type.value_type) + else: + raise ValueError(f"Arrow type {arrow_type} does not have a datasets dtype equivalent.") + + +def string_to_arrow(datasets_dtype: str) -> pa.DataType: + """ + string_to_arrow takes a datasets string dtype and converts it to a pyarrow.DataType. + + In effect, `dt == string_to_arrow(_arrow_to_datasets_dtype(dt))` + + This is necessary because the datasets.Value() primitive type is constructed using a string dtype + + Value(dtype=str) + + But Features.type (via `get_nested_type()` expects to resolve Features into a pyarrow Schema, + which means that each Value() must be able to resolve into a corresponding pyarrow.DataType, which is the + purpose of this function. + """ + + def _dtype_error_msg(dtype, pa_dtype, examples=None, urls=None): + msg = f"{dtype} is not a validly formatted string representation of the pyarrow {pa_dtype} type." + if examples: + examples = ", ".join(examples[:-1]) + " or " + examples[-1] if len(examples) > 1 else examples[0] + msg += f"\nValid examples include: {examples}." + if urls: + urls = ", ".join(urls[:-1]) + " and " + urls[-1] if len(urls) > 1 else urls[0] + msg += f"\nFor more insformation, see: {urls}." + return msg + + if datasets_dtype in pa.__dict__: + return pa.__dict__[datasets_dtype]() + + if (datasets_dtype + "_") in pa.__dict__: + return pa.__dict__[datasets_dtype + "_"]() + + timestamp_matches = re.search(r"^timestamp\[(.*)\]$", datasets_dtype) + if timestamp_matches: + timestamp_internals = timestamp_matches.group(1) + internals_matches = re.search(r"^(s|ms|us|ns),\s*tz=([a-zA-Z0-9/_+\-:]*)$", timestamp_internals) + if timestamp_internals in ["s", "ms", "us", "ns"]: + return pa.timestamp(timestamp_internals) + elif internals_matches: + return pa.timestamp(internals_matches.group(1), internals_matches.group(2)) + else: + raise ValueError( + _dtype_error_msg( + datasets_dtype, + "timestamp", + examples=["timestamp[us]", "timestamp[us, tz=America/New_York"], + urls=["https://arrow.apache.org/docs/python/generated/pyarrow.timestamp.html"], + ) + ) + + duration_matches = re.search(r"^duration\[(.*)\]$", datasets_dtype) + if duration_matches: + duration_internals = duration_matches.group(1) + if duration_internals in ["s", "ms", "us", "ns"]: + return pa.duration(duration_internals) + else: + raise ValueError( + _dtype_error_msg( + datasets_dtype, + "duration", + examples=["duration[s]", "duration[us]"], + urls=["https://arrow.apache.org/docs/python/generated/pyarrow.duration.html"], + ) + ) + + time_matches = re.search(r"^time(.*)\[(.*)\]$", datasets_dtype) + if time_matches: + time_internals_bits = time_matches.group(1) + if time_internals_bits == "32": + time_internals_unit = time_matches.group(2) + if time_internals_unit in ["s", "ms"]: + return pa.time32(time_internals_unit) + else: + raise ValueError( + f"{time_internals_unit} is not a valid unit for the pyarrow time32 type. Supported units: s (second) and ms (millisecond)." + ) + elif time_internals_bits == "64": + time_internals_unit = time_matches.group(2) + if time_internals_unit in ["us", "ns"]: + return pa.time64(time_internals_unit) + else: + raise ValueError( + f"{time_internals_unit} is not a valid unit for the pyarrow time64 type. Supported units: us (microsecond) and ns (nanosecond)." + ) + else: + raise ValueError( + _dtype_error_msg( + datasets_dtype, + "time", + examples=["time32[s]", "time64[us]"], + urls=[ + "https://arrow.apache.org/docs/python/generated/pyarrow.time32.html", + "https://arrow.apache.org/docs/python/generated/pyarrow.time64.html", + ], + ) + ) + + decimal_matches = re.search(r"^decimal(.*)\((.*)\)$", datasets_dtype) + if decimal_matches: + decimal_internals_bits = decimal_matches.group(1) + if decimal_internals_bits == "128": + decimal_internals_precision_and_scale = re.search(r"^(\d+),\s*(-?\d+)$", decimal_matches.group(2)) + if decimal_internals_precision_and_scale: + precision = decimal_internals_precision_and_scale.group(1) + scale = decimal_internals_precision_and_scale.group(2) + return pa.decimal128(int(precision), int(scale)) + else: + raise ValueError( + _dtype_error_msg( + datasets_dtype, + "decimal128", + examples=["decimal128(10, 2)", "decimal128(4, -2)"], + urls=["https://arrow.apache.org/docs/python/generated/pyarrow.decimal128.html"], + ) + ) + elif decimal_internals_bits == "256": + decimal_internals_precision_and_scale = re.search(r"^(\d+),\s*(-?\d+)$", decimal_matches.group(2)) + if decimal_internals_precision_and_scale: + precision = decimal_internals_precision_and_scale.group(1) + scale = decimal_internals_precision_and_scale.group(2) + return pa.decimal256(int(precision), int(scale)) + else: + raise ValueError( + _dtype_error_msg( + datasets_dtype, + "decimal256", + examples=["decimal256(30, 2)", "decimal256(38, -4)"], + urls=["https://arrow.apache.org/docs/python/generated/pyarrow.decimal256.html"], + ) + ) + else: + raise ValueError( + _dtype_error_msg( + datasets_dtype, + "decimal", + examples=["decimal128(12, 3)", "decimal256(40, 6)"], + urls=[ + "https://arrow.apache.org/docs/python/generated/pyarrow.decimal128.html", + "https://arrow.apache.org/docs/python/generated/pyarrow.decimal256.html", + ], + ) + ) + + raise ValueError( + f"Neither {datasets_dtype} nor {datasets_dtype + '_'} seems to be a pyarrow data type. " + f"Please make sure to use a correct data type, see: " + f"https://arrow.apache.org/docs/python/api/datatypes.html#factory-functions" + ) + + +def _cast_to_python_objects(obj: Any, only_1d_for_numpy: bool, optimize_list_casting: bool) -> tuple[Any, bool]: + """ + Cast pytorch/tensorflow/pandas objects to python numpy array/lists. + It works recursively. + + If `optimize_list_casting` is True, to avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be casted. + If the first element needs to be casted, then all the elements of the list will be casted, otherwise they'll stay the same. + This trick allows to cast objects that contain tokenizers outputs without iterating over every single token for example. + + Args: + obj: the object (nested struct) to cast. + only_1d_for_numpy (bool): whether to keep the full multi-dim tensors as multi-dim numpy arrays, or convert them to + nested lists of 1-dimensional numpy arrays. This can be useful to keep only 1-d arrays to instantiate Arrow arrays. + Indeed Arrow only support converting 1-dimensional array values. + optimize_list_casting (bool): whether to optimize list casting by checking the first non-null element to see if it needs to be casted + and if it doesn't, not checking the rest of the list elements. + + Returns: + casted_obj: the casted object + has_changed (bool): True if the object has been changed, False if it is identical + """ + + if config.TF_AVAILABLE and "tensorflow" in sys.modules: + import tensorflow as tf + + if config.TORCH_AVAILABLE and "torch" in sys.modules: + import torch + + if config.JAX_AVAILABLE and "jax" in sys.modules: + import jax.numpy as jnp + + if config.PIL_AVAILABLE and "PIL" in sys.modules: + import PIL.Image + + if config.PDFPLUMBER_AVAILABLE and "pdfplumber" in sys.modules: + import pdfplumber + + if isinstance(obj, np.ndarray): + if obj.ndim == 0: + return obj[()], True + elif not only_1d_for_numpy or obj.ndim == 1: + return obj, False + else: + return ( + [ + _cast_to_python_objects( + x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0] + for x in obj + ], + True, + ) + elif config.TORCH_AVAILABLE and "torch" in sys.modules and isinstance(obj, torch.Tensor): + if obj.dtype == torch.bfloat16: + return _cast_to_python_objects( + obj.detach().to(torch.float).cpu().numpy(), + only_1d_for_numpy=only_1d_for_numpy, + optimize_list_casting=optimize_list_casting, + )[0], True + if obj.ndim == 0: + return obj.detach().cpu().numpy()[()], True + elif not only_1d_for_numpy or obj.ndim == 1: + return obj.detach().cpu().numpy(), True + else: + return ( + [ + _cast_to_python_objects( + x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0] + for x in obj.detach().cpu().numpy() + ], + True, + ) + elif config.TF_AVAILABLE and "tensorflow" in sys.modules and isinstance(obj, tf.Tensor): + if obj.ndim == 0: + return obj.numpy()[()], True + elif not only_1d_for_numpy or obj.ndim == 1: + return obj.numpy(), True + else: + return ( + [ + _cast_to_python_objects( + x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0] + for x in obj.numpy() + ], + True, + ) + elif config.JAX_AVAILABLE and "jax" in sys.modules and isinstance(obj, jnp.ndarray): + if obj.ndim == 0: + return np.asarray(obj)[()], True + elif not only_1d_for_numpy or obj.ndim == 1: + return np.asarray(obj), True + else: + return ( + [ + _cast_to_python_objects( + x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0] + for x in np.asarray(obj) + ], + True, + ) + elif config.PIL_AVAILABLE and "PIL" in sys.modules and isinstance(obj, PIL.Image.Image): + return encode_pil_image(obj), True + elif config.PDFPLUMBER_AVAILABLE and "pdfplumber" in sys.modules and isinstance(obj, pdfplumber.pdf.PDF): + return encode_pdfplumber_pdf(obj), True + elif isinstance(obj, pd.Series): + return ( + _cast_to_python_objects( + obj.tolist(), only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0], + True, + ) + elif isinstance(obj, pd.DataFrame): + return ( + { + key: _cast_to_python_objects( + value, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0] + for key, value in obj.to_dict("series").items() + }, + True, + ) + elif isinstance(obj, pd.Timestamp): + return obj.to_pydatetime(), True + elif isinstance(obj, pd.Timedelta): + return obj.to_pytimedelta(), True + elif isinstance(obj, Mapping): + has_changed = not isinstance(obj, dict) + output = {} + for k, v in obj.items(): + casted_v, has_changed_v = _cast_to_python_objects( + v, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + ) + has_changed |= has_changed_v + output[k] = casted_v + return output if has_changed else obj, has_changed + elif hasattr(obj, "__array__"): + if np.isscalar(obj): + return obj, False + else: + return ( + _cast_to_python_objects( + obj.__array__(), only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0], + True, + ) + elif isinstance(obj, (list, tuple)): + if len(obj) > 0: + for first_elmt in obj: + if _check_non_null_non_empty_recursive(first_elmt): + break + casted_first_elmt, has_changed_first_elmt = _cast_to_python_objects( + first_elmt, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + ) + if has_changed_first_elmt or not optimize_list_casting: + return ( + [ + _cast_to_python_objects( + elmt, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0] + for elmt in obj + ], + True, + ) + else: + if isinstance(obj, (list, tuple)): + return obj, False + else: + return list(obj), True + else: + return obj, False + else: + return obj, False + + +def cast_to_python_objects(obj: Any, only_1d_for_numpy=False, optimize_list_casting=True) -> Any: + """ + Cast numpy/pytorch/tensorflow/pandas objects to python lists. + It works recursively. + + If `optimize_list_casting` is True, To avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be casted. + If the first element needs to be casted, then all the elements of the list will be casted, otherwise they'll stay the same. + This trick allows to cast objects that contain tokenizers outputs without iterating over every single token for example. + + Args: + obj: the object (nested struct) to cast + only_1d_for_numpy (bool, default ``False``): whether to keep the full multi-dim tensors as multi-dim numpy arrays, or convert them to + nested lists of 1-dimensional numpy arrays. This can be useful to keep only 1-d arrays to instantiate Arrow arrays. + Indeed Arrow only support converting 1-dimensional array values. + optimize_list_casting (bool, default ``True``): whether to optimize list casting by checking the first non-null element to see if it needs to be casted + and if it doesn't, not checking the rest of the list elements. + + Returns: + casted_obj: the casted object + """ + return _cast_to_python_objects( + obj, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0] + + +@dataclass +class Value: + """ + Scalar feature value of a particular data type. + + The possible dtypes of `Value` are as follows: + - `null` + - `bool` + - `int8` + - `int16` + - `int32` + - `int64` + - `uint8` + - `uint16` + - `uint32` + - `uint64` + - `float16` + - `float32` (alias float) + - `float64` (alias double) + - `time32[(s|ms)]` + - `time64[(us|ns)]` + - `timestamp[(s|ms|us|ns)]` + - `timestamp[(s|ms|us|ns), tz=(tzstring)]` + - `date32` + - `date64` + - `duration[(s|ms|us|ns)]` + - `decimal128(precision, scale)` + - `decimal256(precision, scale)` + - `binary` + - `large_binary` + - `string` + - `large_string` + + Args: + dtype (`str`): + Name of the data type. + + Example: + + ```py + >>> from datasets import Features + >>> features = Features({'stars': Value(dtype='int32')}) + >>> features + {'stars': Value(dtype='int32', id=None)} + ``` + """ + + dtype: str + id: Optional[str] = None + # Automatically constructed + pa_type: ClassVar[Any] = None + _type: str = field(default="Value", init=False, repr=False) + + def __post_init__(self): + if self.dtype == "double": # fix inferred type + self.dtype = "float64" + if self.dtype == "float": # fix inferred type + self.dtype = "float32" + self.pa_type = string_to_arrow(self.dtype) + + def __call__(self): + return self.pa_type + + def encode_example(self, value): + if pa.types.is_boolean(self.pa_type): + return bool(value) + elif pa.types.is_integer(self.pa_type): + return int(value) + elif pa.types.is_floating(self.pa_type): + return float(value) + elif pa.types.is_string(self.pa_type): + return str(value) + else: + return value + + +class _ArrayXD: + def __post_init__(self): + self.shape = tuple(self.shape) + + def __call__(self): + pa_type = globals()[self.__class__.__name__ + "ExtensionType"](self.shape, self.dtype) + return pa_type + + def encode_example(self, value): + return value + + +@dataclass +class Array2D(_ArrayXD): + """Create a two-dimensional array. + + Args: + shape (`tuple`): + Size of each dimension. + dtype (`str`): + Name of the data type. + + Example: + + ```py + >>> from datasets import Features + >>> features = Features({'x': Array2D(shape=(1, 3), dtype='int32')}) + ``` + """ + + shape: tuple + dtype: str + id: Optional[str] = None + # Automatically constructed + _type: str = field(default="Array2D", init=False, repr=False) + + +@dataclass +class Array3D(_ArrayXD): + """Create a three-dimensional array. + + Args: + shape (`tuple`): + Size of each dimension. + dtype (`str`): + Name of the data type. + + Example: + + ```py + >>> from datasets import Features + >>> features = Features({'x': Array3D(shape=(1, 2, 3), dtype='int32')}) + ``` + """ + + shape: tuple + dtype: str + id: Optional[str] = None + # Automatically constructed + _type: str = field(default="Array3D", init=False, repr=False) + + +@dataclass +class Array4D(_ArrayXD): + """Create a four-dimensional array. + + Args: + shape (`tuple`): + Size of each dimension. + dtype (`str`): + Name of the data type. + + Example: + + ```py + >>> from datasets import Features + >>> features = Features({'x': Array4D(shape=(1, 2, 2, 3), dtype='int32')}) + ``` + """ + + shape: tuple + dtype: str + id: Optional[str] = None + # Automatically constructed + _type: str = field(default="Array4D", init=False, repr=False) + + +@dataclass +class Array5D(_ArrayXD): + """Create a five-dimensional array. + + Args: + shape (`tuple`): + Size of each dimension. + dtype (`str`): + Name of the data type. + + Example: + + ```py + >>> from datasets import Features + >>> features = Features({'x': Array5D(shape=(1, 2, 2, 3, 3), dtype='int32')}) + ``` + """ + + shape: tuple + dtype: str + id: Optional[str] = None + # Automatically constructed + _type: str = field(default="Array5D", init=False, repr=False) + + +class _ArrayXDExtensionType(pa.ExtensionType): + ndims: Optional[int] = None + + def __init__(self, shape: tuple, dtype: str): + if self.ndims is None or self.ndims <= 1: + raise ValueError("You must instantiate an array type with a value for dim that is > 1") + if len(shape) != self.ndims: + raise ValueError(f"shape={shape} and ndims={self.ndims} don't match") + for dim in range(1, self.ndims): + if shape[dim] is None: + raise ValueError(f"Support only dynamic size on first dimension. Got: {shape}") + self.shape = tuple(shape) + self.value_type = dtype + self.storage_dtype = self._generate_dtype(self.value_type) + pa.ExtensionType.__init__(self, self.storage_dtype, f"{self.__class__.__module__}.{self.__class__.__name__}") + + def __arrow_ext_serialize__(self): + return json.dumps((self.shape, self.value_type)).encode() + + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized): + args = json.loads(serialized) + return cls(*args) + + # This was added to pa.ExtensionType in pyarrow >= 13.0.0 + def __reduce__(self): + return self.__arrow_ext_deserialize__, (self.storage_type, self.__arrow_ext_serialize__()) + + def __hash__(self): + return hash((self.__class__, self.shape, self.value_type)) + + def __arrow_ext_class__(self): + return ArrayExtensionArray + + def _generate_dtype(self, dtype): + dtype = string_to_arrow(dtype) + for d in reversed(self.shape): + dtype = pa.list_(dtype) + # Don't specify the size of the list, since fixed length list arrays have issues + # being validated after slicing in pyarrow 0.17.1 + return dtype + + def to_pandas_dtype(self): + return PandasArrayExtensionDtype(self.value_type) + + +class Array2DExtensionType(_ArrayXDExtensionType): + ndims = 2 + + +class Array3DExtensionType(_ArrayXDExtensionType): + ndims = 3 + + +class Array4DExtensionType(_ArrayXDExtensionType): + ndims = 4 + + +class Array5DExtensionType(_ArrayXDExtensionType): + ndims = 5 + + +# Register the extension types for deserialization +pa.register_extension_type(Array2DExtensionType((1, 2), "int64")) +pa.register_extension_type(Array3DExtensionType((1, 2, 3), "int64")) +pa.register_extension_type(Array4DExtensionType((1, 2, 3, 4), "int64")) +pa.register_extension_type(Array5DExtensionType((1, 2, 3, 4, 5), "int64")) + + +def _is_zero_copy_only(pa_type: pa.DataType, unnest: bool = False) -> bool: + """ + When converting a pyarrow array to a numpy array, we must know whether this could be done in zero-copy or not. + This function returns the value of the ``zero_copy_only`` parameter to pass to ``.to_numpy()``, given the type of the pyarrow array. + + # zero copy is available for all primitive types except booleans and temporal types (date, time, timestamp or duration) + # primitive types are types for which the physical representation in arrow and in numpy + # https://github.com/wesm/arrow/blob/c07b9b48cf3e0bbbab493992a492ae47e5b04cad/python/pyarrow/types.pxi#L821 + # see https://arrow.apache.org/docs/python/generated/pyarrow.Array.html#pyarrow.Array.to_numpy + # and https://issues.apache.org/jira/browse/ARROW-2871?jql=text%20~%20%22boolean%20to_numpy%22 + """ + + def _unnest_pa_type(pa_type: pa.DataType) -> pa.DataType: + if pa.types.is_list(pa_type): + return _unnest_pa_type(pa_type.value_type) + return pa_type + + if unnest: + pa_type = _unnest_pa_type(pa_type) + return pa.types.is_primitive(pa_type) and not (pa.types.is_boolean(pa_type) or pa.types.is_temporal(pa_type)) + + +class ArrayExtensionArray(pa.ExtensionArray): + def __array__(self): + zero_copy_only = _is_zero_copy_only(self.storage.type, unnest=True) + return self.to_numpy(zero_copy_only=zero_copy_only) + + def __getitem__(self, i): + return self.storage[i] + + def to_numpy(self, zero_copy_only=True): + storage: pa.ListArray = self.storage + null_mask = storage.is_null().to_numpy(zero_copy_only=False) + + if self.type.shape[0] is not None: + size = 1 + null_indices = np.arange(len(storage))[null_mask] - np.arange(np.sum(null_mask)) + + for i in range(self.type.ndims): + size *= self.type.shape[i] + storage = storage.flatten() + numpy_arr = storage.to_numpy(zero_copy_only=zero_copy_only) + numpy_arr = numpy_arr.reshape(len(self) - len(null_indices), *self.type.shape) + + if len(null_indices): + numpy_arr = np.insert(numpy_arr.astype(np.float64), null_indices, np.nan, axis=0) + + else: + shape = self.type.shape + ndims = self.type.ndims + arrays = [] + first_dim_offsets = np.array([off.as_py() for off in storage.offsets]) + for i, is_null in enumerate(null_mask): + if is_null: + arrays.append(np.nan) + else: + storage_el = storage[i : i + 1] + first_dim = first_dim_offsets[i + 1] - first_dim_offsets[i] + # flatten storage + for _ in range(ndims): + storage_el = storage_el.flatten() + + numpy_arr = storage_el.to_numpy(zero_copy_only=zero_copy_only) + arrays.append(numpy_arr.reshape(first_dim, *shape[1:])) + + if len(np.unique(np.diff(first_dim_offsets))) > 1: + # ragged + numpy_arr = np.empty(len(arrays), dtype=object) + numpy_arr[:] = arrays + else: + numpy_arr = np.array(arrays) + + return numpy_arr + + def to_pylist(self, maps_as_pydicts: Optional[Literal["lossy", "strict"]] = None): + zero_copy_only = _is_zero_copy_only(self.storage.type, unnest=True) + numpy_arr = self.to_numpy(zero_copy_only=zero_copy_only) + if self.type.shape[0] is None and numpy_arr.dtype == object: + return [arr.tolist() for arr in numpy_arr.tolist()] + else: + return numpy_arr.tolist() + + +class PandasArrayExtensionDtype(PandasExtensionDtype): + _metadata = "value_type" + + def __init__(self, value_type: Union["PandasArrayExtensionDtype", np.dtype]): + self._value_type = value_type + + def __from_arrow__(self, array: Union[pa.Array, pa.ChunkedArray]): + if isinstance(array, pa.ChunkedArray): + array = array.type.wrap_array(pa.concat_arrays([chunk.storage for chunk in array.chunks])) + zero_copy_only = _is_zero_copy_only(array.storage.type, unnest=True) + numpy_arr = array.to_numpy(zero_copy_only=zero_copy_only) + return PandasArrayExtensionArray(numpy_arr) + + @classmethod + def construct_array_type(cls): + return PandasArrayExtensionArray + + @property + def type(self) -> type: + return np.ndarray + + @property + def kind(self) -> str: + return "O" + + @property + def name(self) -> str: + return f"array[{self.value_type}]" + + @property + def value_type(self) -> np.dtype: + return self._value_type + + +class PandasArrayExtensionArray(PandasExtensionArray): + def __init__(self, data: np.ndarray, copy: bool = False): + self._data = data if not copy else np.array(data) + self._dtype = PandasArrayExtensionDtype(data.dtype) + + def __array__(self, dtype=None): + """ + Convert to NumPy Array. + Note that Pandas expects a 1D array when dtype is set to object. + But for other dtypes, the returned shape is the same as the one of ``data``. + + More info about pandas 1D requirement for PandasExtensionArray here: + https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.api.extensions.ExtensionArray.html#pandas.api.extensions.ExtensionArray + + """ + if dtype == np.dtype(object): + out = np.empty(len(self._data), dtype=object) + for i in range(len(self._data)): + out[i] = self._data[i] + return out + if dtype is None: + return self._data + else: + return self._data.astype(dtype) + + def copy(self, deep: bool = False) -> "PandasArrayExtensionArray": + return PandasArrayExtensionArray(self._data, copy=True) + + @classmethod + def _from_sequence( + cls, scalars, dtype: Optional[PandasArrayExtensionDtype] = None, copy: bool = False + ) -> "PandasArrayExtensionArray": + if len(scalars) > 1 and all( + isinstance(x, np.ndarray) and x.shape == scalars[0].shape and x.dtype == scalars[0].dtype for x in scalars + ): + data = np.array(scalars, dtype=dtype if dtype is None else dtype.value_type, copy=copy) + else: + data = np.empty(len(scalars), dtype=object) + data[:] = scalars + return cls(data, copy=copy) + + @classmethod + def _concat_same_type(cls, to_concat: Sequence_["PandasArrayExtensionArray"]) -> "PandasArrayExtensionArray": + if len(to_concat) > 1 and all( + va._data.shape == to_concat[0]._data.shape and va._data.dtype == to_concat[0]._data.dtype + for va in to_concat + ): + data = np.vstack([va._data for va in to_concat]) + else: + data = np.empty(len(to_concat), dtype=object) + data[:] = [va._data for va in to_concat] + return cls(data, copy=False) + + @property + def dtype(self) -> PandasArrayExtensionDtype: + return self._dtype + + @property + def nbytes(self) -> int: + return self._data.nbytes + + def isna(self) -> np.ndarray: + return np.array([pd.isna(arr).any() for arr in self._data]) + + def __setitem__(self, key: Union[int, slice, np.ndarray], value: Any) -> None: + raise NotImplementedError() + + def __getitem__(self, item: Union[int, slice, np.ndarray]) -> Union[np.ndarray, "PandasArrayExtensionArray"]: + if isinstance(item, int): + return self._data[item] + return PandasArrayExtensionArray(self._data[item], copy=False) + + def take( + self, indices: Sequence_[int], allow_fill: bool = False, fill_value: bool = None + ) -> "PandasArrayExtensionArray": + indices: np.ndarray = np.asarray(indices, dtype=int) + if allow_fill: + fill_value = ( + self.dtype.na_value if fill_value is None else np.asarray(fill_value, dtype=self.dtype.value_type) + ) + mask = indices == -1 + if (indices < -1).any(): + raise ValueError("Invalid value in `indices`, must be all >= -1 for `allow_fill` is True") + elif len(self) > 0: + pass + elif not np.all(mask): + raise IndexError("Invalid take for empty PandasArrayExtensionArray, must be all -1.") + else: + data = np.array([fill_value] * len(indices), dtype=self.dtype.value_type) + return PandasArrayExtensionArray(data, copy=False) + took = self._data.take(indices, axis=0) + if allow_fill and mask.any(): + took[mask] = [fill_value] * np.sum(mask) + return PandasArrayExtensionArray(took, copy=False) + + def __len__(self) -> int: + return len(self._data) + + def __eq__(self, other) -> np.ndarray: + if not isinstance(other, PandasArrayExtensionArray): + raise NotImplementedError(f"Invalid type to compare to: {type(other)}") + return (self._data == other._data).all() + + +def pandas_types_mapper(dtype): + if isinstance(dtype, _ArrayXDExtensionType): + return PandasArrayExtensionDtype(dtype.value_type) + + +@dataclass +class ClassLabel: + """Feature type for integer class labels. + + There are 3 ways to define a `ClassLabel`, which correspond to the 3 arguments: + + * `num_classes`: Create 0 to (num_classes-1) labels. + * `names`: List of label strings. + * `names_file`: File containing the list of labels. + + Under the hood the labels are stored as integers. + You can use negative integers to represent unknown/missing labels. + + Args: + num_classes (`int`, *optional*): + Number of classes. All labels must be < `num_classes`. + names (`list` of `str`, *optional*): + String names for the integer classes. + The order in which the names are provided is kept. + names_file (`str`, *optional*): + Path to a file with names for the integer classes, one per line. + + Example: + + ```py + >>> from datasets import Features, ClassLabel + >>> features = Features({'label': ClassLabel(num_classes=3, names=['bad', 'ok', 'good'])}) + >>> features + {'label': ClassLabel(names=['bad', 'ok', 'good'], id=None)} + ``` + """ + + num_classes: InitVar[Optional[int]] = None # Pseudo-field: ignored by asdict/fields when converting to/from dict + names: list[str] = None + names_file: InitVar[Optional[str]] = None # Pseudo-field: ignored by asdict/fields when converting to/from dict + id: Optional[str] = None + # Automatically constructed + dtype: ClassVar[str] = "int64" + pa_type: ClassVar[Any] = pa.int64() + _str2int: ClassVar[dict[str, int]] = None + _int2str: ClassVar[dict[int, int]] = None + _type: str = field(default="ClassLabel", init=False, repr=False) + + def __post_init__(self, num_classes, names_file): + self.num_classes = num_classes + self.names_file = names_file + if self.names_file is not None and self.names is not None: + raise ValueError("Please provide either names or names_file but not both.") + # Set self.names + if self.names is None: + if self.names_file is not None: + self.names = self._load_names_from_file(self.names_file) + elif self.num_classes is not None: + self.names = [str(i) for i in range(self.num_classes)] + else: + raise ValueError("Please provide either num_classes, names or names_file.") + elif not isinstance(self.names, SequenceABC): + raise TypeError(f"Please provide names as a list, is {type(self.names)}") + # Set self.num_classes + if self.num_classes is None: + self.num_classes = len(self.names) + elif self.num_classes != len(self.names): + raise ValueError( + "ClassLabel number of names do not match the defined num_classes. " + f"Got {len(self.names)} names VS {self.num_classes} num_classes" + ) + # Prepare mappings + self._int2str = [str(name) for name in self.names] + self._str2int = {name: i for i, name in enumerate(self._int2str)} + if len(self._int2str) != len(self._str2int): + raise ValueError("Some label names are duplicated. Each label name should be unique.") + + def __call__(self): + return self.pa_type + + def str2int(self, values: Union[str, Iterable]) -> Union[int, Iterable]: + """Conversion class name `string` => `integer`. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train") + >>> ds.features["label"].str2int('neg') + 0 + ``` + """ + if not isinstance(values, str) and not isinstance(values, Iterable): + raise ValueError( + f"Values {values} should be a string or an Iterable (list, numpy array, pytorch, tensorflow tensors)" + ) + return_list = True + if isinstance(values, str): + values = [values] + return_list = False + + output = [self._strval2int(value) for value in values] + return output if return_list else output[0] + + def _strval2int(self, value: str) -> int: + failed_parse = False + value = str(value) + # first attempt - raw string value + int_value = self._str2int.get(value) + if int_value is None: + # second attempt - strip whitespace + int_value = self._str2int.get(value.strip()) + if int_value is None: + # third attempt - convert str to int + try: + int_value = int(value) + except ValueError: + failed_parse = True + else: + if int_value < -1 or int_value >= self.num_classes: + failed_parse = True + if failed_parse: + raise ValueError(f"Invalid string class label {value}") + return int_value + + def int2str(self, values: Union[int, Iterable]) -> Union[str, Iterable]: + """Conversion `integer` => class name `string`. + + Regarding unknown/missing labels: passing negative integers raises `ValueError`. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train") + >>> ds.features["label"].int2str(0) + 'neg' + ``` + """ + if not isinstance(values, int) and not isinstance(values, Iterable): + raise ValueError( + f"Values {values} should be an integer or an Iterable (list, numpy array, pytorch, tensorflow tensors)" + ) + return_list = True + if isinstance(values, int): + values = [values] + return_list = False + + for v in values: + if not 0 <= v < self.num_classes: + raise ValueError(f"Invalid integer class label {v:d}") + + output = [self._int2str[int(v)] for v in values] + return output if return_list else output[0] + + def encode_example(self, example_data): + if self.num_classes is None: + raise ValueError( + "Trying to use ClassLabel feature with undefined number of class. " + "Please set ClassLabel.names or num_classes." + ) + + # If a string is given, convert to associated integer + if isinstance(example_data, str): + example_data = self.str2int(example_data) + + # Allowing -1 to mean no label. + if not -1 <= example_data < self.num_classes: + raise ValueError(f"Class label {example_data:d} greater than configured num_classes {self.num_classes}") + return example_data + + def cast_storage(self, storage: Union[pa.StringArray, pa.IntegerArray]) -> pa.Int64Array: + """Cast an Arrow array to the `ClassLabel` arrow storage type. + The Arrow types that can be converted to the `ClassLabel` pyarrow storage type are: + + - `pa.string()` + - `pa.int()` + + Args: + storage (`Union[pa.StringArray, pa.IntegerArray]`): + PyArrow array to cast. + + Returns: + `pa.Int64Array`: Array in the `ClassLabel` arrow storage type. + """ + if isinstance(storage, pa.IntegerArray) and len(storage) > 0: + min_max = pc.min_max(storage).as_py() + if min_max["max"] is not None and min_max["max"] >= self.num_classes: + raise ValueError( + f"Class label {min_max['max']} greater than configured num_classes {self.num_classes}" + ) + elif isinstance(storage, pa.StringArray): + storage = pa.array( + [self._strval2int(label) if label is not None else None for label in storage.to_pylist()] + ) + return array_cast(storage, self.pa_type) + + @staticmethod + def _load_names_from_file(names_filepath): + with open(names_filepath, encoding="utf-8") as f: + return [name.strip() for name in f.read().split("\n") if name.strip()] # Filter empty names + + +@dataclass +class Sequence: + """Construct a list of feature from a single type or a dict of types. + Mostly here for compatiblity with tfds. + + Args: + feature ([`FeatureType`]): + A list of features of a single type or a dictionary of types. + length (`int`): + Length of the sequence. + + Example: + + ```py + >>> from datasets import Features, Sequence, Value, ClassLabel + >>> features = Features({'post': Sequence(feature={'text': Value(dtype='string'), 'upvotes': Value(dtype='int32'), 'label': ClassLabel(num_classes=2, names=['hot', 'cold'])})}) + >>> features + {'post': Sequence(feature={'text': Value(dtype='string', id=None), 'upvotes': Value(dtype='int32', id=None), 'label': ClassLabel(names=['hot', 'cold'], id=None)}, length=-1, id=None)} + ``` + """ + + feature: Any + length: int = -1 + id: Optional[str] = None + # Automatically constructed + dtype: ClassVar[str] = "list" + pa_type: ClassVar[Any] = None + _type: str = field(default="Sequence", init=False, repr=False) + + +@dataclass +class LargeList: + """Feature type for large list data composed of child feature data type. + + It is backed by `pyarrow.LargeListType`, which is like `pyarrow.ListType` but with 64-bit rather than 32-bit offsets. + + Args: + feature ([`FeatureType`]): + Child feature data type of each item within the large list. + """ + + feature: Any + id: Optional[str] = None + # Automatically constructed + pa_type: ClassVar[Any] = None + _type: str = field(default="LargeList", init=False, repr=False) + + +FeatureType = Union[ + dict, + list, + tuple, + Value, + ClassLabel, + Translation, + TranslationVariableLanguages, + LargeList, + Sequence, + Array2D, + Array3D, + Array4D, + Array5D, + Audio, + Image, + Video, + Pdf, +] + + +def _check_non_null_non_empty_recursive(obj, schema: Optional[FeatureType] = None) -> bool: + """ + Check if the object is not None. + If the object is a list or a tuple, recursively check the first element of the sequence and stop if at any point the first element is not a sequence or is an empty sequence. + """ + if obj is None: + return False + elif isinstance(obj, (list, tuple)) and (schema is None or isinstance(schema, (list, tuple, LargeList, Sequence))): + if len(obj) > 0: + if schema is None: + pass + elif isinstance(schema, (list, tuple)): + schema = schema[0] + else: + schema = schema.feature + return _check_non_null_non_empty_recursive(obj[0], schema) + else: + return False + else: + return True + + +def get_nested_type(schema: FeatureType) -> pa.DataType: + """ + get_nested_type() converts a datasets.FeatureType into a pyarrow.DataType, and acts as the inverse of + generate_from_arrow_type(). + + It performs double-duty as the implementation of Features.type and handles the conversion of + datasets.Feature->pa.struct + """ + # Nested structures: we allow dict, list/tuples, sequences + if isinstance(schema, Features): + return pa.struct( + {key: get_nested_type(schema[key]) for key in schema} + ) # Features is subclass of dict, and dict order is deterministic since Python 3.6 + elif isinstance(schema, dict): + return pa.struct( + {key: get_nested_type(schema[key]) for key in schema} + ) # however don't sort on struct types since the order matters + elif isinstance(schema, (list, tuple)): + if len(schema) != 1: + raise ValueError("When defining list feature, you should just provide one example of the inner type") + value_type = get_nested_type(schema[0]) + return pa.list_(value_type) + elif isinstance(schema, LargeList): + value_type = get_nested_type(schema.feature) + return pa.large_list(value_type) + elif isinstance(schema, Sequence): + value_type = get_nested_type(schema.feature) + # We allow to reverse list of dict => dict of list for compatibility with tfds + if isinstance(schema.feature, dict): + data_type = pa.struct({f.name: pa.list_(f.type, schema.length) for f in value_type}) + else: + data_type = pa.list_(value_type, schema.length) + return data_type + + # Other objects are callable which returns their data type (ClassLabel, Array2D, Translation, Arrow datatype creation methods) + return schema() + + +def encode_nested_example(schema, obj, level=0): + """Encode a nested example. + This is used since some features (in particular ClassLabel) have some logic during encoding. + + To avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be encoded. + If the first element needs to be encoded, then all the elements of the list will be encoded, otherwise they'll stay the same. + """ + # Nested structures: we allow dict, list/tuples, sequences + if isinstance(schema, dict): + if level == 0 and obj is None: + raise ValueError("Got None but expected a dictionary instead") + return ( + {k: encode_nested_example(schema[k], obj.get(k), level=level + 1) for k in schema} + if obj is not None + else None + ) + + elif isinstance(schema, (list, tuple)): + sub_schema = schema[0] + if obj is None: + return None + elif isinstance(obj, np.ndarray): + return encode_nested_example(schema, obj.tolist()) + else: + if len(obj) > 0: + for first_elmt in obj: + if _check_non_null_non_empty_recursive(first_elmt, sub_schema): + break + if encode_nested_example(sub_schema, first_elmt, level=level + 1) != first_elmt: + return [encode_nested_example(sub_schema, o, level=level + 1) for o in obj] + return list(obj) + elif isinstance(schema, LargeList): + if obj is None: + return None + else: + if len(obj) > 0: + sub_schema = schema.feature + for first_elmt in obj: + if _check_non_null_non_empty_recursive(first_elmt, sub_schema): + break + if encode_nested_example(sub_schema, first_elmt, level=level + 1) != first_elmt: + return [encode_nested_example(sub_schema, o, level=level + 1) for o in obj] + return list(obj) + elif isinstance(schema, Sequence): + if obj is None: + return None + # We allow to reverse list of dict => dict of list for compatibility with tfds + if isinstance(schema.feature, dict): + # dict of list to fill + list_dict = {} + if isinstance(obj, (list, tuple)): + # obj is a list of dict + for k in schema.feature: + list_dict[k] = [encode_nested_example(schema.feature[k], o.get(k), level=level + 1) for o in obj] + return list_dict + else: + # obj is a single dict + for k in schema.feature: + list_dict[k] = ( + [encode_nested_example(schema.feature[k], o, level=level + 1) for o in obj[k]] + if k in obj + else None + ) + return list_dict + # schema.feature is not a dict + if isinstance(obj, str): # don't interpret a string as a list + raise ValueError(f"Got a string but expected a list instead: '{obj}'") + else: + if len(obj) > 0: + for first_elmt in obj: + if _check_non_null_non_empty_recursive(first_elmt, schema.feature): + break + # be careful when comparing tensors here + if ( + not (isinstance(first_elmt, list) or np.isscalar(first_elmt)) + or encode_nested_example(schema.feature, first_elmt, level=level + 1) != first_elmt + ): + return [encode_nested_example(schema.feature, o, level=level + 1) for o in obj] + return list(obj) + # Object with special encoding: + # ClassLabel will convert from string to int, TranslationVariableLanguages does some checks + elif hasattr(schema, "encode_example"): + return schema.encode_example(obj) if obj is not None else None + # Other object should be directly convertible to a native Arrow type (like Translation and Translation) + return obj + + +def decode_nested_example(schema, obj, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None): + """Decode a nested example. + This is used since some features (in particular Audio and Image) have some logic during decoding. + + To avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be decoded. + If the first element needs to be decoded, then all the elements of the list will be decoded, otherwise they'll stay the same. + """ + # Nested structures: we allow dict, list/tuples, sequences + if isinstance(schema, dict): + return ( + {k: decode_nested_example(sub_schema, sub_obj) for k, (sub_schema, sub_obj) in zip_dict(schema, obj)} + if obj is not None + else None + ) + elif isinstance(schema, (list, tuple)): + sub_schema = schema[0] + if obj is None: + return None + else: + if len(obj) > 0: + for first_elmt in obj: + if _check_non_null_non_empty_recursive(first_elmt, sub_schema): + break + if decode_nested_example(sub_schema, first_elmt) != first_elmt: + return [decode_nested_example(sub_schema, o) for o in obj] + return list(obj) + elif isinstance(schema, LargeList): + if obj is None: + return None + else: + sub_schema = schema.feature + if len(obj) > 0: + for first_elmt in obj: + if _check_non_null_non_empty_recursive(first_elmt, sub_schema): + break + if decode_nested_example(sub_schema, first_elmt) != first_elmt: + return [decode_nested_example(sub_schema, o) for o in obj] + return list(obj) + elif isinstance(schema, Sequence): + # We allow to reverse list of dict => dict of list for compatibility with tfds + if isinstance(schema.feature, dict): + return {k: decode_nested_example([schema.feature[k]], obj[k]) for k in schema.feature} + else: + return decode_nested_example([schema.feature], obj) + # Object with special decoding: + elif hasattr(schema, "decode_example") and getattr(schema, "decode", True): + # we pass the token to read and decode files from private repositories in streaming mode + return schema.decode_example(obj, token_per_repo_id=token_per_repo_id) if obj is not None else None + return obj + + +_FEATURE_TYPES: dict[str, FeatureType] = { + Value.__name__: Value, + ClassLabel.__name__: ClassLabel, + Translation.__name__: Translation, + TranslationVariableLanguages.__name__: TranslationVariableLanguages, + LargeList.__name__: LargeList, + Sequence.__name__: Sequence, + Array2D.__name__: Array2D, + Array3D.__name__: Array3D, + Array4D.__name__: Array4D, + Array5D.__name__: Array5D, + Audio.__name__: Audio, + Image.__name__: Image, + Video.__name__: Video, + Pdf.__name__: Pdf, +} + + +@experimental +def register_feature( + feature_cls: type, + feature_type: str, +): + """ + Register a Feature object using a name and class. + This function must be used on a Feature class. + """ + if feature_type in _FEATURE_TYPES: + logger.warning( + f"Overwriting feature type '{feature_type}' ({_FEATURE_TYPES[feature_type].__name__} -> {feature_cls.__name__})" + ) + _FEATURE_TYPES[feature_type] = feature_cls + + +def generate_from_dict(obj: Any): + """Regenerate the nested feature object from a deserialized dict. + We use the '_type' fields to get the dataclass name to load. + + generate_from_dict is the recursive helper for Features.from_dict, and allows for a convenient constructor syntax + to define features from deserialized JSON dictionaries. This function is used in particular when deserializing + a :class:`DatasetInfo` that was dumped to a JSON object. This acts as an analogue to + :meth:`Features.from_arrow_schema` and handles the recursive field-by-field instantiation, but doesn't require any + mapping to/from pyarrow, except for the fact that it takes advantage of the mapping of pyarrow primitive dtypes + that :class:`Value` automatically performs. + """ + # Nested structures: we allow dict, list/tuples, sequences + if isinstance(obj, list): + return [generate_from_dict(value) for value in obj] + # Otherwise we have a dict or a dataclass + if "_type" not in obj or isinstance(obj["_type"], dict): + return {key: generate_from_dict(value) for key, value in obj.items()} + obj = dict(obj) + _type = obj.pop("_type") + class_type = _FEATURE_TYPES.get(_type, None) or globals().get(_type, None) + + if class_type is None: + raise ValueError(f"Feature type '{_type}' not found. Available feature types: {list(_FEATURE_TYPES.keys())}") + + if class_type == LargeList: + feature = obj.pop("feature") + return LargeList(feature=generate_from_dict(feature), **obj) + if class_type == Sequence: + feature = obj.pop("feature") + return Sequence(feature=generate_from_dict(feature), **obj) + + field_names = {f.name for f in fields(class_type)} + return class_type(**{k: v for k, v in obj.items() if k in field_names}) + + +def generate_from_arrow_type(pa_type: pa.DataType) -> FeatureType: + """ + generate_from_arrow_type accepts an arrow DataType and returns a datasets FeatureType to be used as the type for + a single field. + + This is the high-level arrow->datasets type conversion and is inverted by get_nested_type(). + + This operates at the individual *field* level, whereas Features.from_arrow_schema() operates at the + full schema level and holds the methods that represent the bijection from Features<->pyarrow.Schema + """ + if isinstance(pa_type, pa.StructType): + return {field.name: generate_from_arrow_type(field.type) for field in pa_type} + elif isinstance(pa_type, pa.FixedSizeListType): + return Sequence(feature=generate_from_arrow_type(pa_type.value_type), length=pa_type.list_size) + elif isinstance(pa_type, pa.ListType): + feature = generate_from_arrow_type(pa_type.value_type) + if isinstance(feature, (dict, tuple, list)): + return [feature] + return Sequence(feature=feature) + elif isinstance(pa_type, pa.LargeListType): + feature = generate_from_arrow_type(pa_type.value_type) + return LargeList(feature=feature) + elif isinstance(pa_type, _ArrayXDExtensionType): + array_feature = [None, None, Array2D, Array3D, Array4D, Array5D][pa_type.ndims] + return array_feature(shape=pa_type.shape, dtype=pa_type.value_type) + elif isinstance(pa_type, pa.DataType): + return Value(dtype=_arrow_to_datasets_dtype(pa_type)) + else: + raise ValueError(f"Cannot convert {pa_type} to a Feature type.") + + +def numpy_to_pyarrow_listarray(arr: np.ndarray, type: pa.DataType = None) -> pa.ListArray: + """Build a PyArrow ListArray from a multidimensional NumPy array""" + arr = np.array(arr) + values = pa.array(arr.flatten(), type=type) + for i in range(arr.ndim - 1): + n_offsets = reduce(mul, arr.shape[: arr.ndim - i - 1], 1) + step_offsets = arr.shape[arr.ndim - i - 1] + offsets = pa.array(np.arange(n_offsets + 1) * step_offsets, type=pa.int32()) + values = pa.ListArray.from_arrays(offsets, values) + return values + + +def list_of_pa_arrays_to_pyarrow_listarray(l_arr: list[Optional[pa.Array]]) -> pa.ListArray: + null_mask = np.array([arr is None for arr in l_arr]) + null_indices = np.arange(len(null_mask))[null_mask] - np.arange(np.sum(null_mask)) + l_arr = [arr for arr in l_arr if arr is not None] + offsets = np.cumsum( + [0] + [len(arr) for arr in l_arr], dtype=object + ) # convert to dtype object to allow None insertion + offsets = np.insert(offsets, null_indices, None) + offsets = pa.array(offsets, type=pa.int32()) + values = pa.concat_arrays(l_arr) + return pa.ListArray.from_arrays(offsets, values) + + +def list_of_np_array_to_pyarrow_listarray(l_arr: list[np.ndarray], type: pa.DataType = None) -> pa.ListArray: + """Build a PyArrow ListArray from a possibly nested list of NumPy arrays""" + if len(l_arr) > 0: + return list_of_pa_arrays_to_pyarrow_listarray( + [numpy_to_pyarrow_listarray(arr, type=type) if arr is not None else None for arr in l_arr] + ) + else: + return pa.array([], type=type) + + +def contains_any_np_array(data: Any): + """Return `True` if data is a NumPy ndarray or (recursively) if first non-null value in list is a NumPy ndarray. + + Args: + data (Any): Data. + + Returns: + bool + """ + if isinstance(data, np.ndarray): + return True + elif isinstance(data, list): + return contains_any_np_array(first_non_null_value(data)[1]) + else: + return False + + +def any_np_array_to_pyarrow_listarray(data: Union[np.ndarray, list], type: pa.DataType = None) -> pa.ListArray: + """Convert to PyArrow ListArray either a NumPy ndarray or (recursively) a list that may contain any NumPy ndarray. + + Args: + data (Union[np.ndarray, List]): Data. + type (pa.DataType): Explicit PyArrow DataType passed to coerce the ListArray data type. + + Returns: + pa.ListArray + """ + if isinstance(data, np.ndarray): + return numpy_to_pyarrow_listarray(data, type=type) + elif isinstance(data, list): + return list_of_pa_arrays_to_pyarrow_listarray([any_np_array_to_pyarrow_listarray(i, type=type) for i in data]) + + +def to_pyarrow_listarray(data: Any, pa_type: _ArrayXDExtensionType) -> pa.Array: + """Convert to PyArrow ListArray. + + Args: + data (Any): Sequence, iterable, np.ndarray or pd.Series. + pa_type (_ArrayXDExtensionType): Any of the ArrayNDExtensionType. + + Returns: + pyarrow.Array + """ + if contains_any_np_array(data): + return any_np_array_to_pyarrow_listarray(data, type=pa_type.value_type) + else: + return pa.array(data, pa_type.storage_dtype) + + +def _visit(feature: FeatureType, func: Callable[[FeatureType], Optional[FeatureType]]) -> FeatureType: + """Visit a (possibly nested) feature. + + Args: + feature (FeatureType): the feature type to be checked + Returns: + visited feature (FeatureType) + """ + if isinstance(feature, Features): + out = func(Features({k: _visit(f, func) for k, f in feature.items()})) + elif isinstance(feature, dict): + out = func({k: _visit(f, func) for k, f in feature.items()}) + elif isinstance(feature, (list, tuple)): + out = func([_visit(feature[0], func)]) + elif isinstance(feature, LargeList): + out = func(LargeList(_visit(feature.feature, func))) + elif isinstance(feature, Sequence): + out = func(Sequence(_visit(feature.feature, func), length=feature.length)) + else: + out = func(feature) + return feature if out is None else out + + +_VisitPath = list[Union[str, Literal[0]]] + + +def _visit_with_path( + feature: FeatureType, func: Callable[[FeatureType, _VisitPath], Optional[FeatureType]], visit_path: _VisitPath = [] +) -> FeatureType: + """Visit a (possibly nested) feature with its path in the Feature object. + + A path in a nested feature object is the list of keys that need to be + sequentially accessed to get to the sub-feature. + + For example: + - ["foo"] corresponds to the column "foo" + - ["foo", 0] corresponds to the sub-feature of the lists in "foo" + - ["foo", "bar"] corresponds to the sub-feature of the dicts in "foo" with key "bar" + + Args: + feature (`FeatureType`): the feature type to be checked. + + Returns: + `FeatureType`: the visited feature. + """ + if isinstance(feature, Sequence) and isinstance(feature.feature, dict): + feature = {k: [f] for k, f in feature.feature.items()} + # ^ Sequence of dicts is special, it must be converted to a dict of lists (see https://huggingface.co/docs/datasets/v2.16.1/en/package_reference/main_classes#datasets.Features) + if isinstance(feature, Features): + out = func(Features({k: _visit_with_path(f, func, visit_path + [k]) for k, f in feature.items()}), visit_path) + elif isinstance(feature, dict): + out = func({k: _visit_with_path(f, func, visit_path + [k]) for k, f in feature.items()}, visit_path) + elif isinstance(feature, (list, tuple)): + out = func([_visit_with_path(feature[0], func, visit_path + [0])], visit_path) + elif isinstance(feature, Sequence): + out = func( + Sequence(_visit_with_path(feature.feature, func, visit_path + [0]), length=feature.length), visit_path + ) + elif isinstance(feature, LargeList): + out = func(LargeList(_visit_with_path(feature.feature, func, visit_path + [0])), visit_path) + else: + out = func(feature, visit_path) + return feature if out is None else out + + +def require_decoding(feature: FeatureType, ignore_decode_attribute: bool = False) -> bool: + """Check if a (possibly nested) feature requires decoding. + + Args: + feature (FeatureType): the feature type to be checked + ignore_decode_attribute (:obj:`bool`, default ``False``): Whether to ignore the current value + of the `decode` attribute of the decodable feature types. + Returns: + :obj:`bool` + """ + if isinstance(feature, dict): + return any(require_decoding(f) for f in feature.values()) + elif isinstance(feature, (list, tuple)): + return require_decoding(feature[0]) + elif isinstance(feature, LargeList): + return require_decoding(feature.feature) + elif isinstance(feature, Sequence): + return require_decoding(feature.feature) + else: + return hasattr(feature, "decode_example") and ( + getattr(feature, "decode", True) if not ignore_decode_attribute else True + ) + + +def require_storage_cast(feature: FeatureType) -> bool: + """Check if a (possibly nested) feature requires storage casting. + + Args: + feature (FeatureType): the feature type to be checked + Returns: + :obj:`bool` + """ + if isinstance(feature, dict): + return any(require_storage_cast(f) for f in feature.values()) + elif isinstance(feature, (list, tuple)): + return require_storage_cast(feature[0]) + elif isinstance(feature, LargeList): + return require_storage_cast(feature.feature) + elif isinstance(feature, Sequence): + return require_storage_cast(feature.feature) + else: + return hasattr(feature, "cast_storage") + + +def require_storage_embed(feature: FeatureType) -> bool: + """Check if a (possibly nested) feature requires embedding data into storage. + + Args: + feature (FeatureType): the feature type to be checked + Returns: + :obj:`bool` + """ + if isinstance(feature, dict): + return any(require_storage_cast(f) for f in feature.values()) + elif isinstance(feature, (list, tuple)): + return require_storage_cast(feature[0]) + elif isinstance(feature, LargeList): + return require_storage_cast(feature.feature) + elif isinstance(feature, Sequence): + return require_storage_cast(feature.feature) + else: + return hasattr(feature, "embed_storage") + + +def keep_features_dicts_synced(func): + """ + Wrapper to keep the secondary dictionary, which tracks whether keys are decodable, of the :class:`datasets.Features` object + in sync with the main dictionary. + """ + + @wraps(func) + def wrapper(*args, **kwargs): + if args: + self: "Features" = args[0] + args = args[1:] + else: + self: "Features" = kwargs.pop("self") + out = func(self, *args, **kwargs) + assert hasattr(self, "_column_requires_decoding") + self._column_requires_decoding = {col: require_decoding(feature) for col, feature in self.items()} + return out + + wrapper._decorator_name_ = "_keep_dicts_synced" + return wrapper + + +class Features(dict): + """A special dictionary that defines the internal structure of a dataset. + + Instantiated with a dictionary of type `dict[str, FieldType]`, where keys are the desired column names, + and values are the type of that column. + + `FieldType` can be one of the following: + - [`Value`] feature specifies a single data type value, e.g. `int64` or `string`. + - [`ClassLabel`] feature specifies a predefined set of classes which can have labels associated to them and + will be stored as integers in the dataset. + - Python `dict` specifies a composite feature containing a mapping of sub-fields to sub-features. + It's possible to have nested fields of nested fields in an arbitrary manner. + - Python `list`, [`LargeList`] or [`Sequence`] specifies a composite feature containing a sequence of + sub-features, all of the same feature type. + + + + A [`Sequence`] with an internal dictionary feature will be automatically converted into a dictionary of + lists. This behavior is implemented to have a compatibility layer with the TensorFlow Datasets library but may be + un-wanted in some cases. If you don't want this behavior, you can use a Python `list` or a [`LargeList`] + instead of the [`Sequence`]. + + + + - [`Array2D`], [`Array3D`], [`Array4D`] or [`Array5D`] feature for multidimensional arrays. + - [`Audio`] feature to store the absolute path to an audio file or a dictionary with the relative path + to an audio file ("path" key) and its bytes content ("bytes" key). This feature extracts the audio data. + - [`Image`] feature to store the absolute path to an image file, an `np.ndarray` object, a `PIL.Image.Image` object + or a dictionary with the relative path to an image file ("path" key) and its bytes content ("bytes" key). + This feature extracts the image data. + - [`Translation`] or [`TranslationVariableLanguages`] feature specific to Machine Translation. + """ + + def __init__(*args, **kwargs): + # self not in the signature to allow passing self as a kwarg + if not args: + raise TypeError("descriptor '__init__' of 'Features' object needs an argument") + self, *args = args + super(Features, self).__init__(*args, **kwargs) + self._column_requires_decoding: dict[str, bool] = { + col: require_decoding(feature) for col, feature in self.items() + } + + __setitem__ = keep_features_dicts_synced(dict.__setitem__) + __delitem__ = keep_features_dicts_synced(dict.__delitem__) + update = keep_features_dicts_synced(dict.update) + setdefault = keep_features_dicts_synced(dict.setdefault) + pop = keep_features_dicts_synced(dict.pop) + popitem = keep_features_dicts_synced(dict.popitem) + clear = keep_features_dicts_synced(dict.clear) + + def __reduce__(self): + return Features, (dict(self),) + + @property + def type(self): + """ + Features field types. + + Returns: + :obj:`pyarrow.DataType` + """ + return get_nested_type(self) + + @property + def arrow_schema(self): + """ + Features schema. + + Returns: + :obj:`pyarrow.Schema` + """ + hf_metadata = {"info": {"features": self.to_dict()}} + return pa.schema(self.type).with_metadata({"huggingface": json.dumps(hf_metadata)}) + + @classmethod + def from_arrow_schema(cls, pa_schema: pa.Schema) -> "Features": + """ + Construct [`Features`] from Arrow Schema. + It also checks the schema metadata for Hugging Face Datasets features. + Non-nullable fields are not supported and set to nullable. + + Also, pa.dictionary is not supported and it uses its underlying type instead. + Therefore datasets convert DictionaryArray objects to their actual values. + + Args: + pa_schema (`pyarrow.Schema`): + Arrow Schema. + + Returns: + [`Features`] + """ + # try to load features from the arrow schema metadata + metadata_features = Features() + if pa_schema.metadata is not None and b"huggingface" in pa_schema.metadata: + metadata = json.loads(pa_schema.metadata[b"huggingface"].decode()) + if "info" in metadata and "features" in metadata["info"] and metadata["info"]["features"] is not None: + metadata_features = Features.from_dict(metadata["info"]["features"]) + metadata_features_schema = metadata_features.arrow_schema + obj = { + field.name: ( + metadata_features[field.name] + if field.name in metadata_features and metadata_features_schema.field(field.name) == field + else generate_from_arrow_type(field.type) + ) + for field in pa_schema + } + return cls(**obj) + + @classmethod + def from_dict(cls, dic) -> "Features": + """ + Construct [`Features`] from dict. + + Regenerate the nested feature object from a deserialized dict. + We use the `_type` key to infer the dataclass name of the feature `FieldType`. + + It allows for a convenient constructor syntax + to define features from deserialized JSON dictionaries. This function is used in particular when deserializing + a [`DatasetInfo`] that was dumped to a JSON object. This acts as an analogue to + [`Features.from_arrow_schema`] and handles the recursive field-by-field instantiation, but doesn't require + any mapping to/from pyarrow, except for the fact that it takes advantage of the mapping of pyarrow primitive + dtypes that [`Value`] automatically performs. + + Args: + dic (`dict[str, Any]`): + Python dictionary. + + Returns: + `Features` + + Example:: + >>> Features.from_dict({'_type': {'dtype': 'string', 'id': None, '_type': 'Value'}}) + {'_type': Value(dtype='string', id=None)} + """ + obj = generate_from_dict(dic) + return cls(**obj) + + def to_dict(self): + return asdict(self) + + def _to_yaml_list(self) -> list: + # we compute the YAML list from the dict representation that is used for JSON dump + yaml_data = self.to_dict() + + def simplify(feature: dict) -> dict: + if not isinstance(feature, dict): + raise TypeError(f"Expected a dict but got a {type(feature)}: {feature}") + + for list_type in ["large_list", "list", "sequence"]: + # + # list_type: -> list_type: int32 + # dtype: int32 -> + # + if isinstance(feature.get(list_type), dict) and list(feature[list_type]) == ["dtype"]: + feature[list_type] = feature[list_type]["dtype"] + + # + # list_type: -> list_type: + # struct: -> - name: foo + # - name: foo -> dtype: int32 + # dtype: int32 -> + # + if isinstance(feature.get(list_type), dict) and list(feature[list_type]) == ["struct"]: + feature[list_type] = feature[list_type]["struct"] + + # + # class_label: -> class_label: + # names: -> names: + # - negative -> '0': negative + # - positive -> '1': positive + # + if isinstance(feature.get("class_label"), dict) and isinstance(feature["class_label"].get("names"), list): + # server-side requirement: keys must be strings + feature["class_label"]["names"] = { + str(label_id): label_name for label_id, label_name in enumerate(feature["class_label"]["names"]) + } + return feature + + def to_yaml_inner(obj: Union[dict, list]) -> dict: + if isinstance(obj, dict): + _type = obj.pop("_type", None) + if _type == "LargeList": + _feature = obj.pop("feature") + return simplify({"large_list": to_yaml_inner(_feature), **obj}) + elif _type == "Sequence": + _feature = obj.pop("feature") + return simplify({"sequence": to_yaml_inner(_feature), **obj}) + elif _type == "Value": + return obj + elif _type and not obj: + return {"dtype": camelcase_to_snakecase(_type)} + elif _type: + return {"dtype": simplify({camelcase_to_snakecase(_type): obj})} + else: + return {"struct": [{"name": name, **to_yaml_inner(_feature)} for name, _feature in obj.items()]} + elif isinstance(obj, list): + return simplify({"list": simplify(to_yaml_inner(obj[0]))}) + elif isinstance(obj, tuple): + return to_yaml_inner(list(obj)) + else: + raise TypeError(f"Expected a dict or a list but got {type(obj)}: {obj}") + + def to_yaml_types(obj: dict) -> dict: + if isinstance(obj, dict): + return {k: to_yaml_types(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [to_yaml_types(v) for v in obj] + elif isinstance(obj, tuple): + return to_yaml_types(list(obj)) + else: + return obj + + return to_yaml_types(to_yaml_inner(yaml_data)["struct"]) + + @classmethod + def _from_yaml_list(cls, yaml_data: list) -> "Features": + yaml_data = copy.deepcopy(yaml_data) + + # we convert the list obtained from YAML data into the dict representation that is used for JSON dump + + def unsimplify(feature: dict) -> dict: + if not isinstance(feature, dict): + raise TypeError(f"Expected a dict but got a {type(feature)}: {feature}") + + for list_type in ["large_list", "list", "sequence"]: + # + # list_type: int32 -> list_type: + # -> dtype: int32 + # + if isinstance(feature.get(list_type), str): + feature[list_type] = {"dtype": feature[list_type]} + + # + # class_label: -> class_label: + # names: -> names: + # '0': negative -> - negative + # '1': positive -> - positive + # + if isinstance(feature.get("class_label"), dict) and isinstance(feature["class_label"].get("names"), dict): + label_ids = sorted(feature["class_label"]["names"], key=int) + if label_ids and [int(label_id) for label_id in label_ids] != list(range(int(label_ids[-1]) + 1)): + raise ValueError( + f"ClassLabel expected a value for all label ids [0:{int(label_ids[-1]) + 1}] but some ids are missing." + ) + feature["class_label"]["names"] = [feature["class_label"]["names"][label_id] for label_id in label_ids] + return feature + + def from_yaml_inner(obj: Union[dict, list]) -> Union[dict, list]: + if isinstance(obj, dict): + if not obj: + return {} + _type = next(iter(obj)) + if _type == "large_list": + _feature = unsimplify(obj).pop(_type) + return {"feature": from_yaml_inner(_feature), **obj, "_type": "LargeList"} + if _type == "sequence": + _feature = unsimplify(obj).pop(_type) + return {"feature": from_yaml_inner(_feature), **obj, "_type": "Sequence"} + if _type == "list": + return [from_yaml_inner(unsimplify(obj)[_type])] + if _type == "struct": + return from_yaml_inner(obj["struct"]) + elif _type == "dtype": + if isinstance(obj["dtype"], str): + # e.g. int32, float64, string, audio, image + try: + Value(obj["dtype"]) + return {**obj, "_type": "Value"} + except ValueError: + # e.g. Audio, Image, ArrayXD + return {"_type": snakecase_to_camelcase(obj["dtype"])} + else: + return from_yaml_inner(obj["dtype"]) + else: + return {"_type": snakecase_to_camelcase(_type), **unsimplify(obj)[_type]} + elif isinstance(obj, list): + names = [_feature.pop("name") for _feature in obj] + return {name: from_yaml_inner(_feature) for name, _feature in zip(names, obj)} + else: + raise TypeError(f"Expected a dict or a list but got {type(obj)}: {obj}") + + return cls.from_dict(from_yaml_inner(yaml_data)) + + def encode_example(self, example): + """ + Encode example into a format for Arrow. + + Args: + example (`dict[str, Any]`): + Data in a Dataset row. + + Returns: + `dict[str, Any]` + """ + example = cast_to_python_objects(example) + return encode_nested_example(self, example) + + def encode_column(self, column, column_name: str): + """ + Encode column into a format for Arrow. + + Args: + column (`list[Any]`): + Data in a Dataset column. + column_name (`str`): + Dataset column name. + + Returns: + `list[Any]` + """ + column = cast_to_python_objects(column) + return [encode_nested_example(self[column_name], obj, level=1) for obj in column] + + def encode_batch(self, batch): + """ + Encode batch into a format for Arrow. + + Args: + batch (`dict[str, list[Any]]`): + Data in a Dataset batch. + + Returns: + `dict[str, list[Any]]` + """ + encoded_batch = {} + if set(batch) != set(self): + raise ValueError(f"Column mismatch between batch {set(batch)} and features {set(self)}") + for key, column in batch.items(): + column = cast_to_python_objects(column) + encoded_batch[key] = [encode_nested_example(self[key], obj, level=1) for obj in column] + return encoded_batch + + def decode_example(self, example: dict, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None): + """Decode example with custom feature decoding. + + Args: + example (`dict[str, Any]`): + Dataset row data. + token_per_repo_id (`dict`, *optional*): + To access and decode audio or image files from private repositories on the Hub, you can pass + a dictionary `repo_id (str) -> token (bool or str)`. + + Returns: + `dict[str, Any]` + """ + + return { + column_name: decode_nested_example(feature, value, token_per_repo_id=token_per_repo_id) + if self._column_requires_decoding[column_name] + else value + for column_name, (feature, value) in zip_dict( + {key: value for key, value in self.items() if key in example}, example + ) + } + + def decode_column(self, column: list, column_name: str): + """Decode column with custom feature decoding. + + Args: + column (`list[Any]`): + Dataset column data. + column_name (`str`): + Dataset column name. + + Returns: + `list[Any]` + """ + return ( + [decode_nested_example(self[column_name], value) if value is not None else None for value in column] + if self._column_requires_decoding[column_name] + else column + ) + + def decode_batch(self, batch: dict, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None): + """Decode batch with custom feature decoding. + + Args: + batch (`dict[str, list[Any]]`): + Dataset batch data. + token_per_repo_id (`dict`, *optional*): + To access and decode audio or image files from private repositories on the Hub, you can pass + a dictionary repo_id (str) -> token (bool or str) + + Returns: + `dict[str, list[Any]]` + """ + decoded_batch = {} + for column_name, column in batch.items(): + decoded_batch[column_name] = ( + [ + decode_nested_example(self[column_name], value, token_per_repo_id=token_per_repo_id) + if value is not None + else None + for value in column + ] + if self._column_requires_decoding[column_name] + else column + ) + return decoded_batch + + def copy(self) -> "Features": + """ + Make a deep copy of [`Features`]. + + Returns: + [`Features`] + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train") + >>> copy_of_features = ds.features.copy() + >>> copy_of_features + {'label': ClassLabel(names=['neg', 'pos'], id=None), + 'text': Value(dtype='string', id=None)} + ``` + """ + return copy.deepcopy(self) + + def reorder_fields_as(self, other: "Features") -> "Features": + """ + Reorder Features fields to match the field order of other [`Features`]. + + The order of the fields is important since it matters for the underlying arrow data. + Re-ordering the fields allows to make the underlying arrow data type match. + + Args: + other ([`Features`]): + The other [`Features`] to align with. + + Returns: + [`Features`] + + Example:: + + >>> from datasets import Features, Sequence, Value + >>> # let's say we have two features with a different order of nested fields (for a and b for example) + >>> f1 = Features({"root": Sequence({"a": Value("string"), "b": Value("string")})}) + >>> f2 = Features({"root": {"b": Sequence(Value("string")), "a": Sequence(Value("string"))}}) + >>> assert f1.type != f2.type + >>> # re-ordering keeps the base structure (here Sequence is defined at the root level), but makes the fields order match + >>> f1.reorder_fields_as(f2) + {'root': Sequence(feature={'b': Value(dtype='string', id=None), 'a': Value(dtype='string', id=None)}, length=-1, id=None)} + >>> assert f1.reorder_fields_as(f2).type == f2.type + """ + + def recursive_reorder(source, target, stack=""): + stack_position = " at " + stack[1:] if stack else "" + if isinstance(target, Sequence): + target = target.feature + if isinstance(target, dict): + target = {k: [v] for k, v in target.items()} + else: + target = [target] + if isinstance(source, Sequence): + sequence_kwargs = vars(source).copy() + source = sequence_kwargs.pop("feature") + if isinstance(source, dict): + source = {k: [v] for k, v in source.items()} + reordered = recursive_reorder(source, target, stack) + return Sequence({k: v[0] for k, v in reordered.items()}, **sequence_kwargs) + else: + source = [source] + reordered = recursive_reorder(source, target, stack) + return Sequence(reordered[0], **sequence_kwargs) + elif isinstance(source, dict): + if not isinstance(target, dict): + raise ValueError(f"Type mismatch: between {source} and {target}" + stack_position) + if sorted(source) != sorted(target): + message = ( + f"Keys mismatch: between {source} (source) and {target} (target).\n" + f"{source.keys() - target.keys()} are missing from target " + f"and {target.keys() - source.keys()} are missing from source" + stack_position + ) + raise ValueError(message) + return {key: recursive_reorder(source[key], target[key], stack + f".{key}") for key in target} + elif isinstance(source, list): + if not isinstance(target, list): + raise ValueError(f"Type mismatch: between {source} and {target}" + stack_position) + if len(source) != len(target): + raise ValueError(f"Length mismatch: between {source} and {target}" + stack_position) + return [recursive_reorder(source[i], target[i], stack + ".") for i in range(len(target))] + elif isinstance(source, LargeList): + if not isinstance(target, LargeList): + raise ValueError(f"Type mismatch: between {source} and {target}" + stack_position) + return LargeList(recursive_reorder(source.feature, target.feature, stack)) + else: + return source + + return Features(recursive_reorder(self, other)) + + def flatten(self, max_depth=16) -> "Features": + """Flatten the features. Every dictionary column is removed and is replaced by + all the subfields it contains. The new fields are named by concatenating the + name of the original column and the subfield name like this: `.`. + + If a column contains nested dictionaries, then all the lower-level subfields names are + also concatenated to form new columns: `..`, etc. + + Returns: + [`Features`]: + The flattened features. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("rajpurkar/squad", split="train") + >>> ds.features.flatten() + {'answers.answer_start': Sequence(feature=Value(dtype='int32', id=None), length=-1, id=None), + 'answers.text': Sequence(feature=Value(dtype='string', id=None), length=-1, id=None), + 'context': Value(dtype='string', id=None), + 'id': Value(dtype='string', id=None), + 'question': Value(dtype='string', id=None), + 'title': Value(dtype='string', id=None)} + ``` + """ + for depth in range(1, max_depth): + no_change = True + flattened = self.copy() + for column_name, subfeature in self.items(): + if isinstance(subfeature, dict): + no_change = False + flattened.update({f"{column_name}.{k}": v for k, v in subfeature.items()}) + del flattened[column_name] + elif isinstance(subfeature, Sequence) and isinstance(subfeature.feature, dict): + no_change = False + flattened.update( + { + f"{column_name}.{k}": Sequence(v) if not isinstance(v, dict) else [v] + for k, v in subfeature.feature.items() + } + ) + del flattened[column_name] + elif hasattr(subfeature, "flatten") and subfeature.flatten() != subfeature: + no_change = False + flattened.update({f"{column_name}.{k}": v for k, v in subfeature.flatten().items()}) + del flattened[column_name] + self = flattened + if no_change: + break + return self + + +def _align_features(features_list: list[Features]) -> list[Features]: + """Align dictionaries of features so that the keys that are found in multiple dictionaries share the same feature.""" + name2feature = {} + for features in features_list: + for k, v in features.items(): + if k in name2feature and isinstance(v, dict): + # Recursively align features. + name2feature[k] = _align_features([name2feature[k], v])[0] + elif k not in name2feature or (isinstance(name2feature[k], Value) and name2feature[k].dtype == "null"): + name2feature[k] = v + + return [Features({k: name2feature[k] for k in features.keys()}) for features in features_list] + + +def _check_if_features_can_be_aligned(features_list: list[Features]): + """Check if the dictionaries of features can be aligned. + + Two dictonaries of features can be aligned if the keys they share have the same type or some of them is of type `Value("null")`. + """ + name2feature = {} + for features in features_list: + for k, v in features.items(): + if k not in name2feature or (isinstance(name2feature[k], Value) and name2feature[k].dtype == "null"): + name2feature[k] = v + + for features in features_list: + for k, v in features.items(): + if isinstance(v, dict) and isinstance(name2feature[k], dict): + # Deep checks for structure. + _check_if_features_can_be_aligned([name2feature[k], v]) + elif not (isinstance(v, Value) and v.dtype == "null") and name2feature[k] != v: + raise ValueError( + f'The features can\'t be aligned because the key {k} of features {features} has unexpected type - {v} (expected either {name2feature[k]} or Value("null").' + ) diff --git a/lib/python3.12/site-packages/datasets/features/image.py b/lib/python3.12/site-packages/datasets/features/image.py new file mode 100644 index 0000000000000000000000000000000000000000..de18efd2719e5773a2a6da908f92f062ba86f0ac --- /dev/null +++ b/lib/python3.12/site-packages/datasets/features/image.py @@ -0,0 +1,382 @@ +import os +import sys +import warnings +from dataclasses import dataclass, field +from io import BytesIO +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union + +import numpy as np +import pyarrow as pa + +from .. import config +from ..download.download_config import DownloadConfig +from ..table import array_cast +from ..utils.file_utils import is_local_path, xopen +from ..utils.py_utils import first_non_null_value, no_op_if_value_is_null, string_to_dict + + +if TYPE_CHECKING: + import PIL.Image + + from .features import FeatureType + + +_IMAGE_COMPRESSION_FORMATS: Optional[list[str]] = None +_NATIVE_BYTEORDER = "<" if sys.byteorder == "little" else ">" +# Origin: https://github.com/python-pillow/Pillow/blob/698951e19e19972aeed56df686868f1329981c12/src/PIL/Image.py#L3126 minus "|i1" which values are not preserved correctly when saving and loading an image +_VALID_IMAGE_ARRAY_DTPYES = [ + np.dtype("|b1"), + np.dtype("|u1"), + np.dtype("u2"), + np.dtype("i2"), + np.dtype("u4"), + np.dtype("i4"), + np.dtype("f4"), + np.dtype("f8"), +] + + +@dataclass +class Image: + """Image [`Feature`] to read image data from an image file. + + Input: The Image feature accepts as input: + - A `str`: Absolute path to the image file (i.e. random access is allowed). + - A `dict` with the keys: + + - `path`: String with relative path of the image file to the archive file. + - `bytes`: Bytes of the image file. + + This is useful for archived files with sequential access. + + - An `np.ndarray`: NumPy array representing an image. + - A `PIL.Image.Image`: PIL image object. + + Args: + mode (`str`, *optional*): + The mode to convert the image to. If `None`, the native mode of the image is used. + decode (`bool`, defaults to `True`): + Whether to decode the image data. If `False`, + returns the underlying dictionary in the format `{"path": image_path, "bytes": image_bytes}`. + + Examples: + + ```py + >>> from datasets import load_dataset, Image + >>> ds = load_dataset("AI-Lab-Makerere/beans", split="train") + >>> ds.features["image"] + Image(decode=True, id=None) + >>> ds[0]["image"] + + >>> ds = ds.cast_column('image', Image(decode=False)) + {'bytes': None, + 'path': '/root/.cache/huggingface/datasets/downloads/extracted/b0a21163f78769a2cf11f58dfc767fb458fc7cea5c05dccc0144a2c0f0bc1292/train/healthy/healthy_train.85.jpg'} + ``` + """ + + mode: Optional[str] = None + decode: bool = True + id: Optional[str] = None + # Automatically constructed + dtype: ClassVar[str] = "PIL.Image.Image" + pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()}) + _type: str = field(default="Image", init=False, repr=False) + + def __call__(self): + return self.pa_type + + def encode_example(self, value: Union[str, bytes, bytearray, dict, np.ndarray, "PIL.Image.Image"]) -> dict: + """Encode example into a format for Arrow. + + Args: + value (`str`, `np.ndarray`, `PIL.Image.Image` or `dict`): + Data passed as input to Image feature. + + Returns: + `dict` with "path" and "bytes" fields + """ + if config.PIL_AVAILABLE: + import PIL.Image + else: + raise ImportError("To support encoding images, please install 'Pillow'.") + + if isinstance(value, list): + value = np.array(value) + + if isinstance(value, str): + return {"path": value, "bytes": None} + elif isinstance(value, (bytes, bytearray)): + return {"path": None, "bytes": value} + elif isinstance(value, np.ndarray): + # convert the image array to PNG/TIFF bytes + return encode_np_array(value) + elif isinstance(value, PIL.Image.Image): + # convert the PIL image to bytes (default format is PNG/TIFF) + return encode_pil_image(value) + elif value.get("path") is not None and os.path.isfile(value["path"]): + # we set "bytes": None to not duplicate the data if they're already available locally + return {"bytes": None, "path": value.get("path")} + elif value.get("bytes") is not None or value.get("path") is not None: + # store the image bytes, and path is used to infer the image format using the file extension + return {"bytes": value.get("bytes"), "path": value.get("path")} + else: + raise ValueError( + f"An image sample should have one of 'path' or 'bytes' but they are missing or None in {value}." + ) + + def decode_example(self, value: dict, token_per_repo_id=None) -> "PIL.Image.Image": + """Decode example image file into image data. + + Args: + value (`str` or `dict`): + A string with the absolute image file path, a dictionary with + keys: + + - `path`: String with absolute or relative image file path. + - `bytes`: The bytes of the image file. + token_per_repo_id (`dict`, *optional*): + To access and decode + image files from private repositories on the Hub, you can pass + a dictionary repo_id (`str`) -> token (`bool` or `str`). + + Returns: + `PIL.Image.Image` + """ + if not self.decode: + raise RuntimeError("Decoding is disabled for this feature. Please use Image(decode=True) instead.") + + if config.PIL_AVAILABLE: + import PIL.Image + import PIL.ImageOps + else: + raise ImportError("To support decoding images, please install 'Pillow'.") + + if token_per_repo_id is None: + token_per_repo_id = {} + + path, bytes_ = value["path"], value["bytes"] + if bytes_ is None: + if path is None: + raise ValueError(f"An image should have one of 'path' or 'bytes' but both are None in {value}.") + else: + if is_local_path(path): + image = PIL.Image.open(path) + else: + source_url = path.split("::")[-1] + pattern = ( + config.HUB_DATASETS_URL + if source_url.startswith(config.HF_ENDPOINT) + else config.HUB_DATASETS_HFFS_URL + ) + source_url_fields = string_to_dict(source_url, pattern) + token = ( + token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None + ) + download_config = DownloadConfig(token=token) + with xopen(path, "rb", download_config=download_config) as f: + bytes_ = BytesIO(f.read()) + image = PIL.Image.open(bytes_) + else: + image = PIL.Image.open(BytesIO(bytes_)) + image.load() # to avoid "Too many open files" errors + if image.getexif().get(PIL.Image.ExifTags.Base.Orientation) is not None: + image = PIL.ImageOps.exif_transpose(image) + if self.mode and self.mode != image.mode: + image = image.convert(self.mode) + return image + + def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]: + """If in the decodable state, return the feature itself, otherwise flatten the feature into a dictionary.""" + from .features import Value + + return ( + self + if self.decode + else { + "bytes": Value("binary"), + "path": Value("string"), + } + ) + + def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray, pa.ListArray]) -> pa.StructArray: + """Cast an Arrow array to the Image arrow storage type. + The Arrow types that can be converted to the Image pyarrow storage type are: + + - `pa.string()` - it must contain the "path" data + - `pa.binary()` - it must contain the image bytes + - `pa.struct({"bytes": pa.binary()})` + - `pa.struct({"path": pa.string()})` + - `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter + - `pa.list(*)` - it must contain the image array data + + Args: + storage (`Union[pa.StringArray, pa.StructArray, pa.ListArray]`): + PyArrow array to cast. + + Returns: + `pa.StructArray`: Array in the Image arrow storage type, that is + `pa.struct({"bytes": pa.binary(), "path": pa.string()})`. + """ + if pa.types.is_string(storage.type): + bytes_array = pa.array([None] * len(storage), type=pa.binary()) + storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_binary(storage.type): + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_struct(storage.type): + if storage.type.get_field_index("bytes") >= 0: + bytes_array = storage.field("bytes") + else: + bytes_array = pa.array([None] * len(storage), type=pa.binary()) + if storage.type.get_field_index("path") >= 0: + path_array = storage.field("path") + else: + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_list(storage.type): + bytes_array = pa.array( + [encode_np_array(np.array(arr))["bytes"] if arr is not None else None for arr in storage.to_pylist()], + type=pa.binary(), + ) + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays( + [bytes_array, path_array], ["bytes", "path"], mask=bytes_array.is_null() + ) + return array_cast(storage, self.pa_type) + + def embed_storage(self, storage: pa.StructArray) -> pa.StructArray: + """Embed image files into the Arrow array. + + Args: + storage (`pa.StructArray`): + PyArrow array to embed. + + Returns: + `pa.StructArray`: Array in the Image arrow storage type, that is + `pa.struct({"bytes": pa.binary(), "path": pa.string()})`. + """ + + @no_op_if_value_is_null + def path_to_bytes(path): + with xopen(path, "rb") as f: + bytes_ = f.read() + return bytes_ + + bytes_array = pa.array( + [ + (path_to_bytes(x["path"]) if x["bytes"] is None else x["bytes"]) if x is not None else None + for x in storage.to_pylist() + ], + type=pa.binary(), + ) + path_array = pa.array( + [os.path.basename(path) if path is not None else None for path in storage.field("path").to_pylist()], + type=pa.string(), + ) + storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=bytes_array.is_null()) + return array_cast(storage, self.pa_type) + + +def list_image_compression_formats() -> list[str]: + if config.PIL_AVAILABLE: + import PIL.Image + else: + raise ImportError("To support encoding images, please install 'Pillow'.") + + global _IMAGE_COMPRESSION_FORMATS + if _IMAGE_COMPRESSION_FORMATS is None: + PIL.Image.init() + _IMAGE_COMPRESSION_FORMATS = list(set(PIL.Image.OPEN.keys()) & set(PIL.Image.SAVE.keys())) + return _IMAGE_COMPRESSION_FORMATS + + +def image_to_bytes(image: "PIL.Image.Image") -> bytes: + """Convert a PIL Image object to bytes using native compression if possible, otherwise use PNG/TIFF compression.""" + buffer = BytesIO() + if image.format in list_image_compression_formats(): + format = image.format + else: + format = "PNG" if image.mode in ["1", "L", "LA", "RGB", "RGBA"] else "TIFF" + image.save(buffer, format=format) + return buffer.getvalue() + + +def encode_pil_image(image: "PIL.Image.Image") -> dict: + if hasattr(image, "filename") and image.filename != "": + return {"path": image.filename, "bytes": None} + else: + return {"path": None, "bytes": image_to_bytes(image)} + + +def encode_np_array(array: np.ndarray) -> dict: + if config.PIL_AVAILABLE: + import PIL.Image + else: + raise ImportError("To support encoding images, please install 'Pillow'.") + + dtype = array.dtype + dtype_byteorder = dtype.byteorder if dtype.byteorder != "=" else _NATIVE_BYTEORDER + dtype_kind = dtype.kind + dtype_itemsize = dtype.itemsize + + dest_dtype = None + + # Multi-channel array case (only np.dtype("|u1") is allowed) + if array.shape[2:]: + if dtype_kind not in ["u", "i"]: + raise TypeError( + f"Unsupported array dtype {dtype} for image encoding. Only {dest_dtype} is supported for multi-channel arrays." + ) + dest_dtype = np.dtype("|u1") + if dtype != dest_dtype: + warnings.warn(f"Downcasting array dtype {dtype} to {dest_dtype} to be compatible with 'Pillow'") + # Exact match + elif dtype in _VALID_IMAGE_ARRAY_DTPYES: + dest_dtype = dtype + else: # Downcast the type within the kind (np.can_cast(from_type, to_type, casting="same_kind") doesn't behave as expected, so do it manually) + while dtype_itemsize >= 1: + dtype_str = dtype_byteorder + dtype_kind + str(dtype_itemsize) + if np.dtype(dtype_str) in _VALID_IMAGE_ARRAY_DTPYES: + dest_dtype = np.dtype(dtype_str) + warnings.warn(f"Downcasting array dtype {dtype} to {dest_dtype} to be compatible with 'Pillow'") + break + else: + dtype_itemsize //= 2 + if dest_dtype is None: + raise TypeError( + f"Cannot downcast dtype {dtype} to a valid image dtype. Valid image dtypes: {_VALID_IMAGE_ARRAY_DTPYES}" + ) + + image = PIL.Image.fromarray(array.astype(dest_dtype)) + return {"path": None, "bytes": image_to_bytes(image)} + + +def objects_to_list_of_image_dicts( + objs: Union[list[str], list[dict], list[np.ndarray], list["PIL.Image.Image"]], +) -> list[dict]: + """Encode a list of objects into a format suitable for creating an extension array of type `ImageExtensionType`.""" + if config.PIL_AVAILABLE: + import PIL.Image + else: + raise ImportError("To support encoding images, please install 'Pillow'.") + + if objs: + _, obj = first_non_null_value(objs) + if isinstance(obj, str): + return [{"path": obj, "bytes": None} if obj is not None else None for obj in objs] + if isinstance(obj, np.ndarray): + obj_to_image_dict_func = no_op_if_value_is_null(encode_np_array) + return [obj_to_image_dict_func(obj) for obj in objs] + elif isinstance(obj, PIL.Image.Image): + obj_to_image_dict_func = no_op_if_value_is_null(encode_pil_image) + return [obj_to_image_dict_func(obj) for obj in objs] + else: + return objs + else: + return objs diff --git a/lib/python3.12/site-packages/datasets/features/pdf.py b/lib/python3.12/site-packages/datasets/features/pdf.py new file mode 100644 index 0000000000000000000000000000000000000000..afba910387a886bcb6563986b7e2ab17aa9dcf48 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/features/pdf.py @@ -0,0 +1,238 @@ +import os +from dataclasses import dataclass, field +from io import BytesIO +from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union + +import pyarrow as pa + +from .. import config +from ..download.download_config import DownloadConfig +from ..table import array_cast +from ..utils.file_utils import is_local_path, xopen +from ..utils.py_utils import string_to_dict + + +if TYPE_CHECKING: + import pdfplumber + + from .features import FeatureType + + +def pdf_to_bytes(pdf: "pdfplumber.pdf.PDF") -> bytes: + """Convert a pdfplumber.pdf.PDF object to bytes.""" + with BytesIO() as buffer: + for page in pdf.pages: + buffer.write(page.pdf.stream) + return buffer.getvalue() + + +@dataclass +class Pdf: + """ + **Experimental.** + Pdf [`Feature`] to read pdf documents from a pdf file. + + Input: The Pdf feature accepts as input: + - A `str`: Absolute path to the pdf file (i.e. random access is allowed). + - A `dict` with the keys: + - `path`: String with relative path of the pdf file in a dataset repository. + - `bytes`: Bytes of the pdf file. + This is useful for archived files with sequential access. + + - A `pdfplumber.pdf.PDF`: pdfplumber pdf object. + + Args: + mode (`str`, *optional*): + The mode to convert the pdf to. If `None`, the native mode of the pdf is used. + decode (`bool`, defaults to `True`): + Whether to decode the pdf data. If `False`, + returns the underlying dictionary in the format `{"path": pdf_path, "bytes": pdf_bytes}`. + + Examples: + + ```py + >>> from datasets import Dataset, Pdf + >>> ds = Dataset.from_dict({"pdf": ["path/to/pdf/file.pdf"]}).cast_column("pdf", Pdf()) + >>> ds.features["pdf"] + Pdf(decode=True, id=None) + >>> ds[0]["pdf"] + + >>> ds = ds.cast_column("pdf", Pdf(decode=False)) + >>> ds[0]["pdf"] + {'bytes': None, + 'path': 'path/to/pdf/file.pdf'} + ``` + """ + + decode: bool = True + id: Optional[str] = None + + # Automatically constructed + dtype: ClassVar[str] = "pdfplumber.pdf.PDF" + pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()}) + _type: str = field(default="Pdf", init=False, repr=False) + + def __call__(self): + return self.pa_type + + def encode_example(self, value: Union[str, bytes, bytearray, dict, "pdfplumber.pdf.PDF"]) -> dict: + """Encode example into a format for Arrow. + + Args: + value (`str`, `bytes`, `pdfplumber.pdf.PDF` or `dict`): + Data passed as input to Pdf feature. + + Returns: + `dict` with "path" and "bytes" fields + """ + if config.PDFPLUMBER_AVAILABLE: + import pdfplumber + else: + pdfplumber = None + + if isinstance(value, str): + return {"path": value, "bytes": None} + elif isinstance(value, (bytes, bytearray)): + return {"path": None, "bytes": value} + elif pdfplumber is not None and isinstance(value, pdfplumber.pdf.PDF): + # convert the pdfplumber.pdf.PDF to bytes + return encode_pdfplumber_pdf(value) + elif value.get("path") is not None and os.path.isfile(value["path"]): + # we set "bytes": None to not duplicate the data if they're already available locally + return {"bytes": None, "path": value.get("path")} + elif value.get("bytes") is not None or value.get("path") is not None: + # store the pdf bytes, and path is used to infer the pdf format using the file extension + return {"bytes": value.get("bytes"), "path": value.get("path")} + else: + raise ValueError( + f"A pdf sample should have one of 'path' or 'bytes' but they are missing or None in {value}." + ) + + def decode_example(self, value: dict, token_per_repo_id=None) -> "pdfplumber.pdf.PDF": + """Decode example pdf file into pdf data. + + Args: + value (`str` or `dict`): + A string with the absolute pdf file path, a dictionary with + keys: + + - `path`: String with absolute or relative pdf file path. + - `bytes`: The bytes of the pdf file. + + token_per_repo_id (`dict`, *optional*): + To access and decode pdf files from private repositories on + the Hub, you can pass a dictionary + repo_id (`str`) -> token (`bool` or `str`). + + Returns: + `pdfplumber.pdf.PDF` + """ + if not self.decode: + raise RuntimeError("Decoding is disabled for this feature. Please use Pdf(decode=True) instead.") + + if config.PDFPLUMBER_AVAILABLE: + import pdfplumber + else: + raise ImportError("To support decoding pdfs, please install 'pdfplumber'.") + + if token_per_repo_id is None: + token_per_repo_id = {} + + path, bytes_ = value["path"], value["bytes"] + if bytes_ is None: + if path is None: + raise ValueError(f"A pdf should have one of 'path' or 'bytes' but both are None in {value}.") + else: + if is_local_path(path): + pdf = pdfplumber.open(path) + else: + source_url = path.split("::")[-1] + pattern = ( + config.HUB_DATASETS_URL + if source_url.startswith(config.HF_ENDPOINT) + else config.HUB_DATASETS_HFFS_URL + ) + try: + repo_id = string_to_dict(source_url, pattern)["repo_id"] + token = token_per_repo_id.get(repo_id) + except ValueError: + token = None + download_config = DownloadConfig(token=token) + f = xopen(path, "rb", download_config=download_config) + return pdfplumber.open(f) + else: + with pdfplumber.open(BytesIO(bytes_)) as p: + pdf = p + + return pdf + + def flatten(self) -> Union["FeatureType", Dict[str, "FeatureType"]]: + """If in the decodable state, return the feature itself, otherwise flatten the feature into a dictionary.""" + from .features import Value + + return ( + self + if self.decode + else { + "bytes": Value("binary"), + "path": Value("string"), + } + ) + + def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray, pa.ListArray]) -> pa.StructArray: + """Cast an Arrow array to the Pdf arrow storage type. + The Arrow types that can be converted to the Pdf pyarrow storage type are: + + - `pa.string()` - it must contain the "path" data + - `pa.binary()` - it must contain the image bytes + - `pa.struct({"bytes": pa.binary()})` + - `pa.struct({"path": pa.string()})` + - `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter + - `pa.list(*)` - it must contain the pdf array data + + Args: + storage (`Union[pa.StringArray, pa.StructArray, pa.ListArray]`): + PyArrow array to cast. + + Returns: + `pa.StructArray`: Array in the Pdf arrow storage type, that is + `pa.struct({"bytes": pa.binary(), "path": pa.string()})`. + """ + if pa.types.is_string(storage.type): + bytes_array = pa.array([None] * len(storage), type=pa.binary()) + storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_binary(storage.type): + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_struct(storage.type): + if storage.type.get_field_index("bytes") >= 0: + bytes_array = storage.field("bytes") + else: + bytes_array = pa.array([None] * len(storage), type=pa.binary()) + if storage.type.get_field_index("path") >= 0: + path_array = storage.field("path") + else: + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null()) + return array_cast(storage, self.pa_type) + + +def encode_pdfplumber_pdf(pdf: "pdfplumber.pdf.PDF") -> dict: + """ + Encode a pdfplumber.pdf.PDF object into a dictionary. + + If the PDF has an associated file path, returns the path. Otherwise, serializes + the PDF content into bytes. + + Args: + pdf (pdfplumber.pdf.PDF): A pdfplumber PDF object. + + Returns: + dict: A dictionary with "path" or "bytes" field. + """ + if hasattr(pdf, "stream") and hasattr(pdf.stream, "name") and pdf.stream.name: + # Return the path if the PDF has an associated file path + return {"path": pdf.stream.name, "bytes": None} + else: + # Convert the PDF to bytes if no path is available + return {"path": None, "bytes": pdf_to_bytes(pdf)} diff --git a/lib/python3.12/site-packages/datasets/features/translation.py b/lib/python3.12/site-packages/datasets/features/translation.py new file mode 100644 index 0000000000000000000000000000000000000000..3503fcdf56ba92903abd6248137427d780168dfd --- /dev/null +++ b/lib/python3.12/site-packages/datasets/features/translation.py @@ -0,0 +1,129 @@ +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union + +import pyarrow as pa + + +if TYPE_CHECKING: + from .features import FeatureType + + +@dataclass +class Translation: + """`Feature` for translations with fixed languages per example. + Here for compatibility with tfds. + + Args: + languages (`dict`): + A dictionary for each example mapping string language codes to string translations. + + Example: + + ```python + >>> # At construction time: + >>> datasets.features.Translation(languages=['en', 'fr', 'de']) + >>> # During data generation: + >>> yield { + ... 'en': 'the cat', + ... 'fr': 'le chat', + ... 'de': 'die katze' + ... } + ``` + """ + + languages: list[str] + id: Optional[str] = None + # Automatically constructed + dtype: ClassVar[str] = "dict" + pa_type: ClassVar[Any] = None + _type: str = field(default="Translation", init=False, repr=False) + + def __call__(self): + return pa.struct({lang: pa.string() for lang in sorted(self.languages)}) + + def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]: + """Flatten the Translation feature into a dictionary.""" + from .features import Value + + return {k: Value("string") for k in sorted(self.languages)} + + +@dataclass +class TranslationVariableLanguages: + """`Feature` for translations with variable languages per example. + Here for compatibility with tfds. + + Args: + languages (`dict`): + A dictionary for each example mapping string language codes to one or more string translations. + The languages present may vary from example to example. + + Returns: + - `language` or `translation` (variable-length 1D `tf.Tensor` of `tf.string`): + Language codes sorted in ascending order or plain text translations, sorted to align with language codes. + + Example: + + ```python + >>> # At construction time: + >>> datasets.features.TranslationVariableLanguages(languages=['en', 'fr', 'de']) + >>> # During data generation: + >>> yield { + ... 'en': 'the cat', + ... 'fr': ['le chat', 'la chatte,'] + ... 'de': 'die katze' + ... } + >>> # Tensor returned : + >>> { + ... 'language': ['en', 'de', 'fr', 'fr'], + ... 'translation': ['the cat', 'die katze', 'la chatte', 'le chat'], + ... } + ``` + """ + + languages: Optional[list] = None + num_languages: Optional[int] = None + id: Optional[str] = None + # Automatically constructed + dtype: ClassVar[str] = "dict" + pa_type: ClassVar[Any] = None + _type: str = field(default="TranslationVariableLanguages", init=False, repr=False) + + def __post_init__(self): + self.languages = sorted(set(self.languages)) if self.languages else None + self.num_languages = len(self.languages) if self.languages else None + + def __call__(self): + return pa.struct({"language": pa.list_(pa.string()), "translation": pa.list_(pa.string())}) + + def encode_example(self, translation_dict): + lang_set = set(self.languages) + if set(translation_dict) == {"language", "translation"}: + return translation_dict + elif self.languages and set(translation_dict) - lang_set: + raise ValueError( + f"Some languages in example ({', '.join(sorted(set(translation_dict) - lang_set))}) are not in valid set ({', '.join(lang_set)})." + ) + + # Convert dictionary into tuples, splitting out cases where there are + # multiple translations for a single language. + translation_tuples = [] + for lang, text in translation_dict.items(): + if isinstance(text, str): + translation_tuples.append((lang, text)) + else: + translation_tuples.extend([(lang, el) for el in text]) + + # Ensure translations are in ascending order by language code. + languages, translations = zip(*sorted(translation_tuples)) + + return {"language": languages, "translation": translations} + + def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]: + """Flatten the TranslationVariableLanguages feature into a dictionary.""" + from .features import Sequence, Value + + return { + "language": Sequence(Value("string")), + "translation": Sequence(Value("string")), + } diff --git a/lib/python3.12/site-packages/datasets/features/video.py b/lib/python3.12/site-packages/datasets/features/video.py new file mode 100644 index 0000000000000000000000000000000000000000..e8c5a0e4456fa6309776f0c900828a3521c6115b --- /dev/null +++ b/lib/python3.12/site-packages/datasets/features/video.py @@ -0,0 +1,280 @@ +import os +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, ClassVar, Optional, TypedDict, Union + +import numpy as np +import pyarrow as pa + +from .. import config +from ..download.download_config import DownloadConfig +from ..table import array_cast +from ..utils.file_utils import is_local_path, xopen +from ..utils.py_utils import string_to_dict + + +if TYPE_CHECKING: + from torchvision.io import VideoReader + + from .features import FeatureType + + +class Example(TypedDict): + path: Optional[str] + bytes: Optional[bytes] + + +@dataclass +class Video: + """ + **Experimental.** Video [`Feature`] to read video data from a video file. + + Input: The Video feature accepts as input: + - A `str`: Absolute path to the video file (i.e. random access is allowed). + - A `dict` with the keys: + + - `path`: String with relative path of the video file in a dataset repository. + - `bytes`: Bytes of the video file. + + This is useful for archived files with sequential access. + + - A `torchvision.io.VideoReader`: torchvision video reader object. + + Args: + mode (`str`, *optional*): + The mode to convert the video to. If `None`, the native mode of the video is used. + decode (`bool`, defaults to `True`): + Whether to decode the video data. If `False`, + returns the underlying dictionary in the format `{"path": video_path, "bytes": video_bytes}`. + + Examples: + + ```py + >>> from datasets import Dataset, Video + >>> ds = Dataset.from_dict({"video":["path/to/Screen Recording.mov"]}).cast_column("video", Video()) + >>> ds.features["video"] + Video(decode=True, id=None) + >>> ds[0]["video"] + + >>> ds = ds.cast_column('video', Video(decode=False)) + {'bytes': None, + 'path': 'path/to/Screen Recording.mov'} + ``` + """ + + decode: bool = True + id: Optional[str] = None + # Automatically constructed + dtype: ClassVar[str] = "torchvision.io.VideoReader" + pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()}) + _type: str = field(default="Video", init=False, repr=False) + + def __call__(self): + return self.pa_type + + def encode_example(self, value: Union[str, bytes, bytearray, Example, np.ndarray, "VideoReader"]) -> Example: + """Encode example into a format for Arrow. + + Args: + value (`str`, `np.ndarray`, `VideoReader` or `dict`): + Data passed as input to Video feature. + + Returns: + `dict` with "path" and "bytes" fields + """ + if config.TORCHVISION_AVAILABLE: + from torchvision.io import VideoReader + + else: + VideoReader = None + + if isinstance(value, list): + value = np.array(value) + + if isinstance(value, str): + return {"path": value, "bytes": None} + elif isinstance(value, (bytes, bytearray)): + return {"path": None, "bytes": value} + elif isinstance(value, np.ndarray): + # convert the video array to bytes + return encode_np_array(value) + elif VideoReader is not None and isinstance(value, VideoReader): + # convert the torchvision video reader to bytes + return encode_torchvision_video(value) + elif isinstance(value, dict): + path, bytes_ = value.get("path"), value.get("bytes") + if path is not None and os.path.isfile(path): + # we set "bytes": None to not duplicate the data if they're already available locally + return {"bytes": None, "path": path} + elif bytes_ is not None or path is not None: + # store the video bytes, and path is used to infer the video format using the file extension + return {"bytes": bytes_, "path": path} + else: + raise ValueError( + f"A video sample should have one of 'path' or 'bytes' but they are missing or None in {value}." + ) + else: + raise TypeError(f"Unsupported encode_example type: {type(value)}") + + def decode_example( + self, + value: Union[str, Example], + token_per_repo_id: Optional[dict[str, Union[bool, str]]] = None, + ) -> "VideoReader": + """Decode example video file into video data. + + Args: + value (`str` or `dict`): + A string with the absolute video file path, a dictionary with + keys: + + - `path`: String with absolute or relative video file path. + - `bytes`: The bytes of the video file. + token_per_repo_id (`dict`, *optional*): + To access and decode + video files from private repositories on the Hub, you can pass + a dictionary repo_id (`str`) -> token (`bool` or `str`). + + Returns: + `torchvision.io.VideoReader` + """ + if not self.decode: + raise RuntimeError("Decoding is disabled for this feature. Please use Video(decode=True) instead.") + + if config.TORCHVISION_AVAILABLE: + from torchvision.io import VideoReader + + else: + raise ImportError("To support decoding videos, please install 'torchvision'.") + + if token_per_repo_id is None: + token_per_repo_id = {} + + if isinstance(value, str): + path, bytes_ = value, None + else: + path, bytes_ = value["path"], value["bytes"] + + if bytes_ is None: + if path is None: + raise ValueError(f"A video should have one of 'path' or 'bytes' but both are None in {value}.") + elif is_local_path(path): + video = VideoReader(path) + else: + video = hf_video_reader(path, token_per_repo_id=token_per_repo_id) + else: + video = VideoReader(bytes_) + video._hf_encoded = {"path": path, "bytes": bytes_} + return video + + def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]: + """If in the decodable state, return the feature itself, otherwise flatten the feature into a dictionary.""" + from .features import Value + + return ( + self + if self.decode + else { + "bytes": Value("binary"), + "path": Value("string"), + } + ) + + def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray, pa.ListArray]) -> pa.StructArray: + """Cast an Arrow array to the Video arrow storage type. + The Arrow types that can be converted to the Video pyarrow storage type are: + + - `pa.string()` - it must contain the "path" data + - `pa.binary()` - it must contain the video bytes + - `pa.struct({"bytes": pa.binary()})` + - `pa.struct({"path": pa.string()})` + - `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter + - `pa.list(*)` - it must contain the video array data + + Args: + storage (`Union[pa.StringArray, pa.StructArray, pa.ListArray]`): + PyArrow array to cast. + + Returns: + `pa.StructArray`: Array in the Video arrow storage type, that is + `pa.struct({"bytes": pa.binary(), "path": pa.string()})`. + """ + if pa.types.is_string(storage.type): + bytes_array = pa.array([None] * len(storage), type=pa.binary()) + storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_binary(storage.type): + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_struct(storage.type): + if storage.type.get_field_index("bytes") >= 0: + bytes_array = storage.field("bytes") + else: + bytes_array = pa.array([None] * len(storage), type=pa.binary()) + if storage.type.get_field_index("path") >= 0: + path_array = storage.field("path") + else: + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_list(storage.type): + bytes_array = pa.array( + [encode_np_array(np.array(arr))["bytes"] if arr is not None else None for arr in storage.to_pylist()], + type=pa.binary(), + ) + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays( + [bytes_array, path_array], ["bytes", "path"], mask=bytes_array.is_null() + ) + return array_cast(storage, self.pa_type) + + +def video_to_bytes(video: "VideoReader") -> bytes: + """Convert a torchvision Video object to bytes using native compression if possible""" + raise NotImplementedError() + + +def encode_torchvision_video(video: "VideoReader") -> Example: + if hasattr(video, "_hf_encoded"): + return video._hf_encoded + else: + raise NotImplementedError( + "Encoding a VideoReader that doesn't come from datasets.Video.decode() is not implemented" + ) + + +def encode_np_array(array: np.ndarray) -> Example: + raise NotImplementedError() + + +# Patching torchvision a little bit to: +# 1. store the encoded video data {"path": ..., "bytes": ...} in `video._hf_encoded`` +# 2. add support for hf:// files +# This doesn't affect the normal usage of torchvision. + + +def hf_video_reader( + path: str, token_per_repo_id: Optional[dict[str, Union[bool, str]]] = None, stream: str = "video" +) -> "VideoReader": + import av + from torchvision import get_video_backend + from torchvision.io import VideoReader + + # Load the file from HF + if token_per_repo_id is None: + token_per_repo_id = {} + source_url = path.split("::")[-1] + pattern = config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL + source_url_fields = string_to_dict(source_url, pattern) + token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None + download_config = DownloadConfig(token=token) + f = xopen(path, "rb", download_config=download_config) + + # Instantiate the VideoReader + vr = object.__new__(VideoReader) + vr.backend = get_video_backend() + if vr.backend != "pyav": + raise RuntimeError(f"Unsupported video backend for VideoReader from HF files: {vr.backend}") + vr.container = av.open(f, metadata_errors="ignore") + stream_type = stream.split(":")[0] + stream_id = 0 if len(stream.split(":")) == 1 else int(stream.split(":")[1]) + vr.pyav_stream = {stream_type: stream_id} + vr._c = vr.container.decode(**vr.pyav_stream) + return vr diff --git a/lib/python3.12/site-packages/datasets/filesystems/__init__.py b/lib/python3.12/site-packages/datasets/filesystems/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d19d58bca1fd8f48e2ef9e5fcdfe17f7738692ac --- /dev/null +++ b/lib/python3.12/site-packages/datasets/filesystems/__init__.py @@ -0,0 +1,47 @@ +import importlib +import shutil +import warnings +from typing import List + +import fsspec +import fsspec.asyn +from fsspec.implementations.local import LocalFileSystem + +from . import compression + + +COMPRESSION_FILESYSTEMS: list[compression.BaseCompressedFileFileSystem] = [ + compression.Bz2FileSystem, + compression.GzipFileSystem, + compression.Lz4FileSystem, + compression.XzFileSystem, + compression.ZstdFileSystem, +] + +# Register custom filesystems +for fs_class in COMPRESSION_FILESYSTEMS: + if fs_class.protocol in fsspec.registry and fsspec.registry[fs_class.protocol] is not fs_class: + warnings.warn(f"A filesystem protocol was already set for {fs_class.protocol} and will be overwritten.") + fsspec.register_implementation(fs_class.protocol, fs_class, clobber=True) + + +def is_remote_filesystem(fs: fsspec.AbstractFileSystem) -> bool: + """ + Checks if `fs` is a remote filesystem. + + Args: + fs (`fsspec.spec.AbstractFileSystem`): + An abstract super-class for pythonic file-systems, e.g. `fsspec.filesystem(\'file\')` or `s3fs.S3FileSystem`. + """ + return not isinstance(fs, LocalFileSystem) + + +def rename(fs: fsspec.AbstractFileSystem, src: str, dst: str): + """ + Renames the file `src` in `fs` to `dst`. + """ + if not is_remote_filesystem(fs): + # LocalFileSystem.mv does copy + rm, it is more efficient to simply move a local directory + shutil.move(fs._strip_protocol(src), fs._strip_protocol(dst)) + else: + fs.mv(src, dst, recursive=True) diff --git a/lib/python3.12/site-packages/datasets/filesystems/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/filesystems/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a17e438e52c26e77e01520ef97a720a062ba34f5 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/filesystems/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/filesystems/__pycache__/compression.cpython-312.pyc b/lib/python3.12/site-packages/datasets/filesystems/__pycache__/compression.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b10edd2199ef1e89fdaf8dd11e860ae2833a0339 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/filesystems/__pycache__/compression.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/filesystems/compression.py b/lib/python3.12/site-packages/datasets/filesystems/compression.py new file mode 100644 index 0000000000000000000000000000000000000000..416d5b15f822dcfbaf0d37cac42ee61d3c0734d2 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/filesystems/compression.py @@ -0,0 +1,127 @@ +import os +from functools import partial +from typing import Optional + +import fsspec +from fsspec.archive import AbstractArchiveFileSystem + + +class BaseCompressedFileFileSystem(AbstractArchiveFileSystem): + """Read contents of compressed file as a filesystem with one file inside.""" + + root_marker = "" + protocol: str = ( + None # protocol passed in prefix to the url. ex: "gzip", for gzip://file.txt::http://foo.bar/file.txt.gz + ) + compression: str = None # compression type in fsspec. ex: "gzip" + extension: str = None # extension of the filename to strip. ex: "".gz" to get file.txt from file.txt.gz + + def __init__( + self, fo: str = "", target_protocol: Optional[str] = None, target_options: Optional[dict] = None, **kwargs + ): + """ + The compressed file system can be instantiated from any compressed file. + It reads the contents of compressed file as a filesystem with one file inside, as if it was an archive. + + The single file inside the filesystem is named after the compresssed file, + without the compression extension at the end of the filename. + + Args: + fo (:obj:``str``): Path to compressed file. Will fetch file using ``fsspec.open()`` + mode (:obj:``str``): Currently, only 'rb' accepted + target_protocol(:obj:``str``, optional): To override the FS protocol inferred from a URL. + target_options (:obj:``dict``, optional): Kwargs passed when instantiating the target FS. + """ + super().__init__(self, **kwargs) + self.fo = fo.__fspath__() if hasattr(fo, "__fspath__") else fo + # always open as "rb" since fsspec can then use the TextIOWrapper to make it work for "r" mode + self._open_with_fsspec = partial( + fsspec.open, + self.fo, + mode="rb", + protocol=target_protocol, + compression=self.compression, + client_kwargs={ + "requote_redirect_url": False, # see https://github.com/huggingface/datasets/pull/5459 + "trust_env": True, # Enable reading proxy env variables. + **(target_options or {}).pop("client_kwargs", {}), # To avoid issues if it was already passed. + }, + **(target_options or {}), + ) + self.compressed_name = os.path.basename(self.fo.split("::")[0]) + self.uncompressed_name = ( + self.compressed_name[: self.compressed_name.rindex(".")] + if "." in self.compressed_name + else self.compressed_name + ) + self.dir_cache = None + + @classmethod + def _strip_protocol(cls, path): + # compressed file paths are always relative to the archive root + return super()._strip_protocol(path).lstrip("/") + + def _get_dirs(self): + if self.dir_cache is None: + f = {**self._open_with_fsspec().fs.info(self.fo), "name": self.uncompressed_name} + self.dir_cache = {f["name"]: f} + + def cat(self, path: str): + with self._open_with_fsspec().open() as f: + return f.read() + + def _open( + self, + path: str, + mode: str = "rb", + block_size=None, + autocommit=True, + cache_options=None, + **kwargs, + ): + path = self._strip_protocol(path) + if mode != "rb": + raise ValueError(f"Tried to read with mode {mode} on file {self.fo} opened with mode 'rb'") + return self._open_with_fsspec().open() + + +class Bz2FileSystem(BaseCompressedFileFileSystem): + """Read contents of BZ2 file as a filesystem with one file inside.""" + + protocol = "bz2" + compression = "bz2" + extension = ".bz2" + + +class GzipFileSystem(BaseCompressedFileFileSystem): + """Read contents of GZIP file as a filesystem with one file inside.""" + + protocol = "gzip" + compression = "gzip" + extension = ".gz" + + +class Lz4FileSystem(BaseCompressedFileFileSystem): + """Read contents of LZ4 file as a filesystem with one file inside.""" + + protocol = "lz4" + compression = "lz4" + extension = ".lz4" + + +class XzFileSystem(BaseCompressedFileFileSystem): + """Read contents of .xz (LZMA) file as a filesystem with one file inside.""" + + protocol = "xz" + compression = "xz" + extension = ".xz" + + +class ZstdFileSystem(BaseCompressedFileFileSystem): + """ + Read contents of .zstd file as a filesystem with one file inside. + """ + + protocol = "zstd" + compression = "zstd" + extension = ".zst" diff --git a/lib/python3.12/site-packages/datasets/formatting/__init__.py b/lib/python3.12/site-packages/datasets/formatting/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e29421df342f172cece9e28ff619c7760b34c6a0 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/formatting/__init__.py @@ -0,0 +1,136 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, List, Optional, Type + +from .. import config +from ..utils import logging +from .formatting import ( + ArrowFormatter, + CustomFormatter, + Formatter, + PandasFormatter, + PythonFormatter, + TableFormatter, + TensorFormatter, + format_table, + query_table, +) +from .np_formatter import NumpyFormatter + + +logger = logging.get_logger(__name__) + +_FORMAT_TYPES: dict[Optional[str], type[Formatter]] = {} +_FORMAT_TYPES_ALIASES: dict[Optional[str], str] = {} +_FORMAT_TYPES_ALIASES_UNAVAILABLE: dict[Optional[str], Exception] = {} + + +def _register_formatter( + formatter_cls: type, + format_type: Optional[str], + aliases: Optional[list[str]] = None, +): + """ + Register a Formatter object using a name and optional aliases. + This function must be used on a Formatter class. + """ + aliases = aliases if aliases is not None else [] + if format_type in _FORMAT_TYPES: + logger.warning( + f"Overwriting format type '{format_type}' ({_FORMAT_TYPES[format_type].__name__} -> {formatter_cls.__name__})" + ) + _FORMAT_TYPES[format_type] = formatter_cls + for alias in set(aliases + [format_type]): + if alias in _FORMAT_TYPES_ALIASES: + logger.warning( + f"Overwriting format type alias '{alias}' ({_FORMAT_TYPES_ALIASES[alias]} -> {format_type})" + ) + _FORMAT_TYPES_ALIASES[alias] = format_type + + +def _register_unavailable_formatter( + unavailable_error: Exception, format_type: Optional[str], aliases: Optional[list[str]] = None +): + """ + Register an unavailable Formatter object using a name and optional aliases. + This function must be used on an Exception object that is raised when trying to get the unavailable formatter. + """ + aliases = aliases if aliases is not None else [] + for alias in set(aliases + [format_type]): + _FORMAT_TYPES_ALIASES_UNAVAILABLE[alias] = unavailable_error + + +# Here we define all the available formatting functions that can be used by `Dataset.set_format` +_register_formatter(PythonFormatter, None, aliases=["python"]) +_register_formatter(ArrowFormatter, "arrow", aliases=["pa", "pyarrow"]) +_register_formatter(NumpyFormatter, "numpy", aliases=["np"]) +_register_formatter(PandasFormatter, "pandas", aliases=["pd"]) +_register_formatter(CustomFormatter, "custom") + +if config.POLARS_AVAILABLE: + from .polars_formatter import PolarsFormatter + + _register_formatter(PolarsFormatter, "polars", aliases=["pl"]) +else: + _polars_error = ValueError("Polars needs to be installed to be able to return Polars dataframes.") + _register_unavailable_formatter(_polars_error, "polars", aliases=["pl"]) + +if config.TORCH_AVAILABLE: + from .torch_formatter import TorchFormatter + + _register_formatter(TorchFormatter, "torch", aliases=["pt", "pytorch"]) +else: + _torch_error = ValueError("PyTorch needs to be installed to be able to return PyTorch tensors.") + _register_unavailable_formatter(_torch_error, "torch", aliases=["pt", "pytorch"]) + +if config.TF_AVAILABLE: + from .tf_formatter import TFFormatter + + _register_formatter(TFFormatter, "tensorflow", aliases=["tf"]) +else: + _tf_error = ValueError("Tensorflow needs to be installed to be able to return Tensorflow tensors.") + _register_unavailable_formatter(_tf_error, "tensorflow", aliases=["tf"]) + +if config.JAX_AVAILABLE: + from .jax_formatter import JaxFormatter + + _register_formatter(JaxFormatter, "jax", aliases=[]) +else: + _jax_error = ValueError("JAX needs to be installed to be able to return JAX arrays.") + _register_unavailable_formatter(_jax_error, "jax", aliases=[]) + + +def get_format_type_from_alias(format_type: Optional[str]) -> Optional[str]: + """If the given format type is a known alias, then return its main type name. Otherwise return the type with no change.""" + if format_type in _FORMAT_TYPES_ALIASES: + return _FORMAT_TYPES_ALIASES[format_type] + else: + return format_type + + +def get_formatter(format_type: Optional[str], **format_kwargs) -> Formatter: + """ + Factory function to get a Formatter given its type name and keyword arguments. + A formatter is an object that extracts and formats data from pyarrow table. + It defines the formatting for rows, columns and batches. + If the formatter for a given type name doesn't exist or is not available, an error is raised. + """ + format_type = get_format_type_from_alias(format_type) + if format_type in _FORMAT_TYPES: + return _FORMAT_TYPES[format_type](**format_kwargs) + if format_type in _FORMAT_TYPES_ALIASES_UNAVAILABLE: + raise _FORMAT_TYPES_ALIASES_UNAVAILABLE[format_type] + else: + raise ValueError(f"Format type should be one of {list(_FORMAT_TYPES.keys())}, but got '{format_type}'") diff --git a/lib/python3.12/site-packages/datasets/formatting/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/formatting/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bc442598e68e41fb7361a3756882f4c0dd315062 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/formatting/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/formatting/__pycache__/formatting.cpython-312.pyc b/lib/python3.12/site-packages/datasets/formatting/__pycache__/formatting.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..effb0cbdb4379d9f9451b1e5126d1c3e0b73d7c1 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/formatting/__pycache__/formatting.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/formatting/__pycache__/jax_formatter.cpython-312.pyc b/lib/python3.12/site-packages/datasets/formatting/__pycache__/jax_formatter.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..84827d2e53e9e5d82907680996133b2fcb46ed79 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/formatting/__pycache__/jax_formatter.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/formatting/__pycache__/np_formatter.cpython-312.pyc b/lib/python3.12/site-packages/datasets/formatting/__pycache__/np_formatter.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b6f2a8a340a8eac54a194613e3d7ebf6090c783e Binary files /dev/null and b/lib/python3.12/site-packages/datasets/formatting/__pycache__/np_formatter.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/formatting/__pycache__/polars_formatter.cpython-312.pyc b/lib/python3.12/site-packages/datasets/formatting/__pycache__/polars_formatter.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a0d956e8ecb81a06cd5c9891fc79be7247ca2256 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/formatting/__pycache__/polars_formatter.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/formatting/__pycache__/tf_formatter.cpython-312.pyc b/lib/python3.12/site-packages/datasets/formatting/__pycache__/tf_formatter.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ddbe8a7bcbf308fc20d59b85af71b34b3f07aac9 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/formatting/__pycache__/tf_formatter.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/formatting/__pycache__/torch_formatter.cpython-312.pyc b/lib/python3.12/site-packages/datasets/formatting/__pycache__/torch_formatter.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..52ac233d6f6aa59070afa357d725cb495a3148b7 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/formatting/__pycache__/torch_formatter.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/formatting/formatting.py b/lib/python3.12/site-packages/datasets/formatting/formatting.py new file mode 100644 index 0000000000000000000000000000000000000000..f8fc92c9d1f1df0b344071763bcddc5e7175f33c --- /dev/null +++ b/lib/python3.12/site-packages/datasets/formatting/formatting.py @@ -0,0 +1,673 @@ +# Copyright 2020 The HuggingFace Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import operator +from collections.abc import Iterable, Mapping, MutableMapping +from functools import partial + +# Lint as: python3 +from typing import Any, Callable, Generic, Optional, TypeVar, Union + +import numpy as np +import pandas as pd +import pyarrow as pa + +from ..features import Features +from ..features.features import _ArrayXDExtensionType, _is_zero_copy_only, decode_nested_example, pandas_types_mapper +from ..table import Table +from ..utils.py_utils import no_op_if_value_is_null + + +T = TypeVar("T") + +RowFormat = TypeVar("RowFormat") +ColumnFormat = TypeVar("ColumnFormat") +BatchFormat = TypeVar("BatchFormat") + + +def _is_range_contiguous(key: range) -> bool: + return key.step == 1 and key.stop >= key.start + + +def _raise_bad_key_type(key: Any): + raise TypeError( + f"Wrong key type: '{key}' of type '{type(key)}'. Expected one of int, slice, range, str or Iterable." + ) + + +def _query_table_with_indices_mapping( + table: Table, key: Union[int, slice, range, str, Iterable], indices: Table +) -> pa.Table: + """ + Query a pyarrow Table to extract the subtable that correspond to the given key. + The :obj:`indices` parameter corresponds to the indices mapping in case we cant to take into + account a shuffling or an indices selection for example. + The indices table must contain one column named "indices" of type uint64. + """ + if isinstance(key, int): + key = indices.fast_slice(key % indices.num_rows, 1).column(0)[0].as_py() + return _query_table(table, key) + if isinstance(key, slice): + key = range(*key.indices(indices.num_rows)) + if isinstance(key, range): + if _is_range_contiguous(key) and key.start >= 0: + return _query_table( + table, [i.as_py() for i in indices.fast_slice(key.start, key.stop - key.start).column(0)] + ) + else: + pass # treat as an iterable + if isinstance(key, str): + table = table.select([key]) + return _query_table(table, indices.column(0).to_pylist()) + if isinstance(key, Iterable): + return _query_table(table, [indices.fast_slice(i, 1).column(0)[0].as_py() for i in key]) + + _raise_bad_key_type(key) + + +def _query_table(table: Table, key: Union[int, slice, range, str, Iterable]) -> pa.Table: + """ + Query a pyarrow Table to extract the subtable that correspond to the given key. + """ + if isinstance(key, int): + return table.fast_slice(key % table.num_rows, 1) + if isinstance(key, slice): + key = range(*key.indices(table.num_rows)) + if isinstance(key, range): + if _is_range_contiguous(key) and key.start >= 0: + return table.fast_slice(key.start, key.stop - key.start) + else: + pass # treat as an iterable + if isinstance(key, str): + return table.table.drop([column for column in table.column_names if column != key]) + if isinstance(key, Iterable): + key = np.fromiter(key, np.int64) + if len(key) == 0: + return table.table.slice(0, 0) + # don't use pyarrow.Table.take even for pyarrow >=1.0 (see https://issues.apache.org/jira/browse/ARROW-9773) + return table.fast_gather(key % table.num_rows) + + _raise_bad_key_type(key) + + +def _is_array_with_nulls(pa_array: pa.Array) -> bool: + return pa_array.null_count > 0 + + +class BaseArrowExtractor(Generic[RowFormat, ColumnFormat, BatchFormat]): + """ + Arrow extractor are used to extract data from pyarrow tables. + It makes it possible to extract rows, columns and batches. + These three extractions types have to be implemented. + """ + + def extract_row(self, pa_table: pa.Table) -> RowFormat: + raise NotImplementedError + + def extract_column(self, pa_table: pa.Table) -> ColumnFormat: + raise NotImplementedError + + def extract_batch(self, pa_table: pa.Table) -> BatchFormat: + raise NotImplementedError + + +def _unnest(py_dict: dict[str, list[T]]) -> dict[str, T]: + """Return the first element of a batch (dict) as a row (dict)""" + return {key: array[0] for key, array in py_dict.items()} + + +class SimpleArrowExtractor(BaseArrowExtractor[pa.Table, pa.Array, pa.Table]): + def extract_row(self, pa_table: pa.Table) -> pa.Table: + return pa_table + + def extract_column(self, pa_table: pa.Table) -> pa.Array: + return pa_table.column(0) + + def extract_batch(self, pa_table: pa.Table) -> pa.Table: + return pa_table + + +class PythonArrowExtractor(BaseArrowExtractor[dict, list, dict]): + def extract_row(self, pa_table: pa.Table) -> dict: + return _unnest(pa_table.to_pydict()) + + def extract_column(self, pa_table: pa.Table) -> list: + return pa_table.column(0).to_pylist() + + def extract_batch(self, pa_table: pa.Table) -> dict: + return pa_table.to_pydict() + + +class NumpyArrowExtractor(BaseArrowExtractor[dict, np.ndarray, dict]): + def __init__(self, **np_array_kwargs): + self.np_array_kwargs = np_array_kwargs + + def extract_row(self, pa_table: pa.Table) -> dict: + return _unnest(self.extract_batch(pa_table)) + + def extract_column(self, pa_table: pa.Table) -> np.ndarray: + return self._arrow_array_to_numpy(pa_table[pa_table.column_names[0]]) + + def extract_batch(self, pa_table: pa.Table) -> dict: + return {col: self._arrow_array_to_numpy(pa_table[col]) for col in pa_table.column_names} + + def _arrow_array_to_numpy(self, pa_array: pa.Array) -> np.ndarray: + if isinstance(pa_array, pa.ChunkedArray): + if isinstance(pa_array.type, _ArrayXDExtensionType): + # don't call to_pylist() to preserve dtype of the fixed-size array + zero_copy_only = _is_zero_copy_only(pa_array.type.storage_dtype, unnest=True) + array: list = [ + row for chunk in pa_array.chunks for row in chunk.to_numpy(zero_copy_only=zero_copy_only) + ] + else: + zero_copy_only = _is_zero_copy_only(pa_array.type) and all( + not _is_array_with_nulls(chunk) for chunk in pa_array.chunks + ) + array: list = [ + row for chunk in pa_array.chunks for row in chunk.to_numpy(zero_copy_only=zero_copy_only) + ] + else: + if isinstance(pa_array.type, _ArrayXDExtensionType): + # don't call to_pylist() to preserve dtype of the fixed-size array + zero_copy_only = _is_zero_copy_only(pa_array.type.storage_dtype, unnest=True) + array: list = pa_array.to_numpy(zero_copy_only=zero_copy_only) + else: + zero_copy_only = _is_zero_copy_only(pa_array.type) and not _is_array_with_nulls(pa_array) + array: list = pa_array.to_numpy(zero_copy_only=zero_copy_only).tolist() + + if len(array) > 0: + if any( + (isinstance(x, np.ndarray) and (x.dtype == object or x.shape != array[0].shape)) + or (isinstance(x, float) and np.isnan(x)) + for x in array + ): + if np.lib.NumpyVersion(np.__version__) >= "2.0.0b1": + return np.asarray(array, dtype=object) + return np.array(array, copy=False, dtype=object) + if np.lib.NumpyVersion(np.__version__) >= "2.0.0b1": + return np.asarray(array) + else: + return np.array(array, copy=False) + + +class PandasArrowExtractor(BaseArrowExtractor[pd.DataFrame, pd.Series, pd.DataFrame]): + def extract_row(self, pa_table: pa.Table) -> pd.DataFrame: + return pa_table.slice(length=1).to_pandas(types_mapper=pandas_types_mapper) + + def extract_column(self, pa_table: pa.Table) -> pd.Series: + return pa_table.select([0]).to_pandas(types_mapper=pandas_types_mapper)[pa_table.column_names[0]] + + def extract_batch(self, pa_table: pa.Table) -> pd.DataFrame: + return pa_table.to_pandas(types_mapper=pandas_types_mapper) + + +class PythonFeaturesDecoder: + def __init__( + self, features: Optional[Features], token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None + ): + self.features = features + self.token_per_repo_id = token_per_repo_id + + def decode_row(self, row: dict) -> dict: + return self.features.decode_example(row, token_per_repo_id=self.token_per_repo_id) if self.features else row + + def decode_column(self, column: list, column_name: str) -> list: + return self.features.decode_column(column, column_name) if self.features else column + + def decode_batch(self, batch: dict) -> dict: + return self.features.decode_batch(batch) if self.features else batch + + +class PandasFeaturesDecoder: + def __init__(self, features: Optional[Features]): + self.features = features + + def decode_row(self, row: pd.DataFrame) -> pd.DataFrame: + decode = ( + { + column_name: no_op_if_value_is_null(partial(decode_nested_example, feature)) + for column_name, feature in self.features.items() + if self.features._column_requires_decoding[column_name] + } + if self.features + else {} + ) + if decode: + row[list(decode.keys())] = row.transform(decode) + return row + + def decode_column(self, column: pd.Series, column_name: str) -> pd.Series: + decode = ( + no_op_if_value_is_null(partial(decode_nested_example, self.features[column_name])) + if self.features and column_name in self.features and self.features._column_requires_decoding[column_name] + else None + ) + if decode: + column = column.transform(decode) + return column + + def decode_batch(self, batch: pd.DataFrame) -> pd.DataFrame: + return self.decode_row(batch) + + +class LazyDict(MutableMapping): + """A dictionary backed by Arrow data. The values are formatted on-the-fly when accessing the dictionary.""" + + def __init__(self, pa_table: pa.Table, formatter: "Formatter"): + self.pa_table = pa_table + self.formatter = formatter + + self.data = dict.fromkeys(pa_table.column_names) + self.keys_to_format = set(self.data.keys()) + + def __len__(self): + return len(self.data) + + def __getitem__(self, key): + value = self.data[key] + if key in self.keys_to_format: + value = self.format(key) + self.data[key] = value + self.keys_to_format.remove(key) + return value + + def __setitem__(self, key, value): + if key in self.keys_to_format: + self.keys_to_format.remove(key) + self.data[key] = value + + def __delitem__(self, key) -> None: + if key in self.keys_to_format: + self.keys_to_format.remove(key) + del self.data[key] + + def __iter__(self): + return iter(self.data) + + def __contains__(self, key): + return key in self.data + + def __repr__(self): + self._format_all() + return repr(self.data) + + def __or__(self, other): + if isinstance(other, LazyDict): + inst = self.copy() + other = other.copy() + other._format_all() + inst.keys_to_format -= other.data.keys() + inst.data = inst.data | other.data + return inst + if isinstance(other, dict): + inst = self.copy() + inst.keys_to_format -= other.keys() + inst.data = inst.data | other + return inst + return NotImplemented + + def __ror__(self, other): + if isinstance(other, LazyDict): + inst = self.copy() + other = other.copy() + other._format_all() + inst.keys_to_format -= other.data.keys() + inst.data = other.data | inst.data + return inst + if isinstance(other, dict): + inst = self.copy() + inst.keys_to_format -= other.keys() + inst.data = other | inst.data + return inst + return NotImplemented + + def __ior__(self, other): + if isinstance(other, LazyDict): + other = other.copy() + other._format_all() + self.keys_to_format -= other.data.keys() + self.data |= other.data + else: + self.keys_to_format -= other.keys() + self.data |= other + return self + + def __copy__(self): + # Identical to `UserDict.__copy__` + inst = self.__class__.__new__(self.__class__) + inst.__dict__.update(self.__dict__) + # Create a copy and avoid triggering descriptors + inst.__dict__["data"] = self.__dict__["data"].copy() + inst.__dict__["keys_to_format"] = self.__dict__["keys_to_format"].copy() + return inst + + def copy(self): + import copy + + return copy.copy(self) + + @classmethod + def fromkeys(cls, iterable, value=None): + raise NotImplementedError + + def format(self, key): + raise NotImplementedError + + def _format_all(self): + for key in self.keys_to_format: + self.data[key] = self.format(key) + self.keys_to_format.clear() + + +class LazyRow(LazyDict): + def format(self, key): + return self.formatter.format_column(self.pa_table.select([key]))[0] + + +class LazyBatch(LazyDict): + def format(self, key): + return self.formatter.format_column(self.pa_table.select([key])) + + +class Formatter(Generic[RowFormat, ColumnFormat, BatchFormat]): + """ + A formatter is an object that extracts and formats data from pyarrow tables. + It defines the formatting for rows, columns and batches. + """ + + simple_arrow_extractor = SimpleArrowExtractor + python_arrow_extractor = PythonArrowExtractor + numpy_arrow_extractor = NumpyArrowExtractor + pandas_arrow_extractor = PandasArrowExtractor + + def __init__( + self, + features: Optional[Features] = None, + token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None, + ): + self.features = features + self.token_per_repo_id = token_per_repo_id + self.python_features_decoder = PythonFeaturesDecoder(self.features, self.token_per_repo_id) + self.pandas_features_decoder = PandasFeaturesDecoder(self.features) + + def __call__(self, pa_table: pa.Table, query_type: str) -> Union[RowFormat, ColumnFormat, BatchFormat]: + if query_type == "row": + return self.format_row(pa_table) + elif query_type == "column": + return self.format_column(pa_table) + elif query_type == "batch": + return self.format_batch(pa_table) + + def format_row(self, pa_table: pa.Table) -> RowFormat: + raise NotImplementedError + + def format_column(self, pa_table: pa.Table) -> ColumnFormat: + raise NotImplementedError + + def format_batch(self, pa_table: pa.Table) -> BatchFormat: + raise NotImplementedError + + +class TensorFormatter(Formatter[RowFormat, ColumnFormat, BatchFormat]): + def recursive_tensorize(self, data_struct: dict): + raise NotImplementedError + + +class TableFormatter(Formatter[RowFormat, ColumnFormat, BatchFormat]): + table_type: str + column_type: str + + +class ArrowFormatter(TableFormatter[pa.Table, pa.Array, pa.Table]): + table_type = "arrow table" + column_type = "arrow array" + + def format_row(self, pa_table: pa.Table) -> pa.Table: + return self.simple_arrow_extractor().extract_row(pa_table) + + def format_column(self, pa_table: pa.Table) -> pa.Array: + return self.simple_arrow_extractor().extract_column(pa_table) + + def format_batch(self, pa_table: pa.Table) -> pa.Table: + return self.simple_arrow_extractor().extract_batch(pa_table) + + +class PythonFormatter(Formatter[Mapping, list, Mapping]): + def __init__(self, features=None, lazy=False, token_per_repo_id=None): + super().__init__(features, token_per_repo_id) + self.lazy = lazy + + def format_row(self, pa_table: pa.Table) -> Mapping: + if self.lazy: + return LazyRow(pa_table, self) + row = self.python_arrow_extractor().extract_row(pa_table) + row = self.python_features_decoder.decode_row(row) + return row + + def format_column(self, pa_table: pa.Table) -> list: + column = self.python_arrow_extractor().extract_column(pa_table) + column = self.python_features_decoder.decode_column(column, pa_table.column_names[0]) + return column + + def format_batch(self, pa_table: pa.Table) -> Mapping: + if self.lazy: + return LazyBatch(pa_table, self) + batch = self.python_arrow_extractor().extract_batch(pa_table) + batch = self.python_features_decoder.decode_batch(batch) + return batch + + +class PandasFormatter(TableFormatter[pd.DataFrame, pd.Series, pd.DataFrame]): + table_type = "pandas dataframe" + column_type = "pandas series" + + def format_row(self, pa_table: pa.Table) -> pd.DataFrame: + row = self.pandas_arrow_extractor().extract_row(pa_table) + row = self.pandas_features_decoder.decode_row(row) + return row + + def format_column(self, pa_table: pa.Table) -> pd.Series: + column = self.pandas_arrow_extractor().extract_column(pa_table) + column = self.pandas_features_decoder.decode_column(column, pa_table.column_names[0]) + return column + + def format_batch(self, pa_table: pa.Table) -> pd.DataFrame: + row = self.pandas_arrow_extractor().extract_batch(pa_table) + row = self.pandas_features_decoder.decode_batch(row) + return row + + +class CustomFormatter(Formatter[dict, ColumnFormat, dict]): + """ + A user-defined custom formatter function defined by a ``transform``. + The transform must take as input a batch of data extracted for an arrow table using the python extractor, + and return a batch. + If the output batch is not a dict, then output_all_columns won't work. + If the output batch has several fields, then querying a single column won't work since we don't know which field + to return. + """ + + def __init__(self, transform: Callable[[dict], dict], features=None, token_per_repo_id=None, **kwargs): + super().__init__(features=features, token_per_repo_id=token_per_repo_id) + self.transform = transform + + def format_row(self, pa_table: pa.Table) -> dict: + formatted_batch = self.format_batch(pa_table) + try: + return _unnest(formatted_batch) + except Exception as exc: + raise TypeError( + f"Custom formatting function must return a dict of sequences to be able to pick a row, but got {formatted_batch}" + ) from exc + + def format_column(self, pa_table: pa.Table) -> ColumnFormat: + formatted_batch = self.format_batch(pa_table) + if hasattr(formatted_batch, "keys"): + if len(formatted_batch.keys()) > 1: + raise TypeError( + "Tried to query a column but the custom formatting function returns too many columns. " + f"Only one column was expected but got columns {list(formatted_batch.keys())}." + ) + else: + raise TypeError( + f"Custom formatting function must return a dict to be able to pick a row, but got {formatted_batch}" + ) + try: + return formatted_batch[pa_table.column_names[0]] + except Exception as exc: + raise TypeError( + f"Custom formatting function must return a dict to be able to pick a row, but got {formatted_batch}" + ) from exc + + def format_batch(self, pa_table: pa.Table) -> dict: + batch = self.python_arrow_extractor().extract_batch(pa_table) + batch = self.python_features_decoder.decode_batch(batch) + return self.transform(batch) + + +def _check_valid_column_key(key: str, columns: list[str]) -> None: + if key not in columns: + raise KeyError(f"Column {key} not in the dataset. Current columns in the dataset: {columns}") + + +def _check_valid_index_key(key: Union[int, slice, range, Iterable], size: int) -> None: + if isinstance(key, int): + if (key < 0 and key + size < 0) or (key >= size): + raise IndexError(f"Invalid key: {key} is out of bounds for size {size}") + return + elif isinstance(key, slice): + pass + elif isinstance(key, range): + if len(key) > 0: + _check_valid_index_key(max(key), size=size) + _check_valid_index_key(min(key), size=size) + elif isinstance(key, Iterable): + if len(key) > 0: + _check_valid_index_key(int(max(key)), size=size) + _check_valid_index_key(int(min(key)), size=size) + else: + _raise_bad_key_type(key) + + +def key_to_query_type(key: Union[int, slice, range, str, Iterable]) -> str: + if isinstance(key, int): + return "row" + elif isinstance(key, str): + return "column" + elif isinstance(key, (slice, range, Iterable)): + return "batch" + _raise_bad_key_type(key) + + +def query_table( + table: Table, + key: Union[int, slice, range, str, Iterable], + indices: Optional[Table] = None, +) -> pa.Table: + """ + Query a Table to extract the subtable that correspond to the given key. + + Args: + table (``datasets.table.Table``): The input Table to query from + key (``Union[int, slice, range, str, Iterable]``): The key can be of different types: + - an integer i: the subtable containing only the i-th row + - a slice [i:j:k]: the subtable containing the rows that correspond to this slice + - a range(i, j, k): the subtable containing the rows that correspond to this range + - a string c: the subtable containing all the rows but only the column c + - an iterable l: the subtable that is the concatenation of all the i-th rows for all i in the iterable + indices (Optional ``datasets.table.Table``): If not None, it is used to re-map the given key to the table rows. + The indices table must contain one column named "indices" of type uint64. + This is used in case of shuffling or rows selection. + + + Returns: + ``pyarrow.Table``: the result of the query on the input table + """ + # Check if key is valid + if not isinstance(key, (int, slice, range, str, Iterable)): + try: + key = operator.index(key) + except TypeError: + _raise_bad_key_type(key) + if isinstance(key, str): + _check_valid_column_key(key, table.column_names) + else: + size = indices.num_rows if indices is not None else table.num_rows + _check_valid_index_key(key, size) + # Query the main table + if indices is None: + pa_subtable = _query_table(table, key) + else: + pa_subtable = _query_table_with_indices_mapping(table, key, indices=indices) + return pa_subtable + + +def format_table( + table: Table, + key: Union[int, slice, range, str, Iterable], + formatter: Formatter, + format_columns: Optional[list] = None, + output_all_columns=False, +): + """ + Format a Table depending on the key that was used and a Formatter object. + + Args: + table (``datasets.table.Table``): The input Table to format + key (``Union[int, slice, range, str, Iterable]``): Depending on the key that was used, the formatter formats + the table as either a row, a column or a batch. + formatter (``datasets.formatting.formatting.Formatter``): Any subclass of a Formatter such as + PythonFormatter, NumpyFormatter, etc. + format_columns (:obj:`List[str]`, optional): if not None, it defines the columns that will be formatted using the + given formatter. Other columns are discarded (unless ``output_all_columns`` is True) + output_all_columns (:obj:`bool`, defaults to False). If True, the formatted output is completed using the columns + that are not in the ``format_columns`` list. For these columns, the PythonFormatter is used. + + + Returns: + A row, column or batch formatted object defined by the Formatter: + - the PythonFormatter returns a dictionary for a row or a batch, and a list for a column. + - the NumpyFormatter returns a dictionary for a row or a batch, and a np.array for a column. + - the PandasFormatter returns a pd.DataFrame for a row or a batch, and a pd.Series for a column. + - the TorchFormatter returns a dictionary for a row or a batch, and a torch.Tensor for a column. + - the TFFormatter returns a dictionary for a row or a batch, and a tf.Tensor for a column. + """ + if isinstance(table, Table): + pa_table = table.table + else: + pa_table = table + query_type = key_to_query_type(key) + python_formatter = PythonFormatter(features=formatter.features) + if format_columns is None: + return formatter(pa_table, query_type=query_type) + elif query_type == "column": + if key in format_columns: + return formatter(pa_table, query_type) + else: + return python_formatter(pa_table, query_type=query_type) + else: + pa_table_to_format = pa_table.drop(col for col in pa_table.column_names if col not in format_columns) + formatted_output = formatter(pa_table_to_format, query_type=query_type) + if output_all_columns: + if isinstance(formatted_output, MutableMapping): + pa_table_with_remaining_columns = pa_table.drop( + col for col in pa_table.column_names if col in format_columns + ) + remaining_columns_dict = python_formatter(pa_table_with_remaining_columns, query_type=query_type) + formatted_output.update(remaining_columns_dict) + else: + raise TypeError( + f"Custom formatting function must return a dict to work with output_all_columns=True, but got {formatted_output}" + ) + return formatted_output diff --git a/lib/python3.12/site-packages/datasets/formatting/jax_formatter.py b/lib/python3.12/site-packages/datasets/formatting/jax_formatter.py new file mode 100644 index 0000000000000000000000000000000000000000..ad414279ac83bd7638d7c74c751e5fffb032bcea --- /dev/null +++ b/lib/python3.12/site-packages/datasets/formatting/jax_formatter.py @@ -0,0 +1,166 @@ +# Copyright 2021 The HuggingFace Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +import sys +from collections.abc import Mapping +from typing import TYPE_CHECKING, Optional + +import numpy as np +import pyarrow as pa + +from .. import config +from ..utils.logging import get_logger +from ..utils.py_utils import map_nested +from .formatting import TensorFormatter + + +if TYPE_CHECKING: + import jax + import jaxlib + +logger = get_logger() + +DEVICE_MAPPING: Optional[dict] = None + + +class JaxFormatter(TensorFormatter[Mapping, "jax.Array", Mapping]): + def __init__(self, features=None, device=None, token_per_repo_id=None, **jnp_array_kwargs): + super().__init__(features=features, token_per_repo_id=token_per_repo_id) + import jax + from jaxlib.xla_client import Device + + if isinstance(device, Device): + raise ValueError( + f"Expected {device} to be a `str` not {type(device)}, as `jaxlib.xla_extension.Device` " + "is not serializable neither with `pickle` nor with `dill`. Instead you can surround " + "the device with `str()` to get its string identifier that will be internally mapped " + "to the actual `jaxlib.xla_extension.Device`." + ) + self.device = device if isinstance(device, str) else str(jax.devices()[0]) + # using global variable since `jaxlib.xla_extension.Device` is not serializable neither + # with `pickle` nor with `dill`, so we need to use a global variable instead + global DEVICE_MAPPING + if DEVICE_MAPPING is None: + DEVICE_MAPPING = self._map_devices_to_str() + if self.device not in list(DEVICE_MAPPING.keys()): + logger.warning( + f"Device with string identifier {self.device} not listed among the available " + f"devices: {list(DEVICE_MAPPING.keys())}, so falling back to the default " + f"device: {str(jax.devices()[0])}." + ) + self.device = str(jax.devices()[0]) + self.jnp_array_kwargs = jnp_array_kwargs + + @staticmethod + def _map_devices_to_str() -> dict[str, "jaxlib.xla_extension.Device"]: + import jax + + return {str(device): device for device in jax.devices()} + + def _consolidate(self, column): + import jax + import jax.numpy as jnp + + if isinstance(column, list) and column: + if all( + isinstance(x, jax.Array) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column + ): + return jnp.stack(column, axis=0) + return column + + def _tensorize(self, value): + import jax + import jax.numpy as jnp + + if isinstance(value, (str, bytes, type(None))): + return value + elif isinstance(value, (np.character, np.ndarray)) and np.issubdtype(value.dtype, np.character): + return value.tolist() + + default_dtype = {} + + if isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.integer): + # the default int precision depends on the jax config + # see https://jax.readthedocs.io/en/latest/notebooks/Common_Gotchas_in_JAX.html#double-64bit-precision + if jax.config.jax_enable_x64: + default_dtype = {"dtype": jnp.int64} + else: + default_dtype = {"dtype": jnp.int32} + elif isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.floating): + default_dtype = {"dtype": jnp.float32} + + if config.PIL_AVAILABLE and "PIL" in sys.modules: + import PIL.Image + + if isinstance(value, PIL.Image.Image): + value = np.asarray(value) + if config.TORCHVISION_AVAILABLE and "torchvision" in sys.modules: + from torchvision.io import VideoReader + + if isinstance(value, VideoReader): + return value # TODO(QL): set output to jax arrays ? + + # using global variable since `jaxlib.xla_extension.Device` is not serializable neither + # with `pickle` nor with `dill`, so we need to use a global variable instead + global DEVICE_MAPPING + if DEVICE_MAPPING is None: + DEVICE_MAPPING = self._map_devices_to_str() + + with jax.default_device(DEVICE_MAPPING[self.device]): + # calling jnp.array on a np.ndarray does copy the data + # see https://github.com/google/jax/issues/4486 + return jnp.array(value, **{**default_dtype, **self.jnp_array_kwargs}) + + def _recursive_tensorize(self, data_struct): + import jax + + # support for torch, tf, jax etc. + if config.TORCH_AVAILABLE and "torch" in sys.modules: + import torch + + if isinstance(data_struct, torch.Tensor): + return self._tensorize(data_struct.detach().cpu().numpy()[()]) + if hasattr(data_struct, "__array__") and not isinstance(data_struct, jax.Array): + data_struct = data_struct.__array__() + # support for nested types like struct of list of struct + if isinstance(data_struct, np.ndarray): + if data_struct.dtype == object: # jax arrays cannot be instantied from an array of objects + return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct]) + elif isinstance(data_struct, (list, tuple)): + return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct]) + return self._tensorize(data_struct) + + def recursive_tensorize(self, data_struct: dict): + return map_nested(self._recursive_tensorize, data_struct, map_list=False) + + def format_row(self, pa_table: pa.Table) -> Mapping: + row = self.numpy_arrow_extractor().extract_row(pa_table) + row = self.python_features_decoder.decode_row(row) + return self.recursive_tensorize(row) + + def format_column(self, pa_table: pa.Table) -> "jax.Array": + column = self.numpy_arrow_extractor().extract_column(pa_table) + column = self.python_features_decoder.decode_column(column, pa_table.column_names[0]) + column = self.recursive_tensorize(column) + column = self._consolidate(column) + return column + + def format_batch(self, pa_table: pa.Table) -> Mapping: + batch = self.numpy_arrow_extractor().extract_batch(pa_table) + batch = self.python_features_decoder.decode_batch(batch) + batch = self.recursive_tensorize(batch) + for column_name in batch: + batch[column_name] = self._consolidate(batch[column_name]) + return batch diff --git a/lib/python3.12/site-packages/datasets/formatting/np_formatter.py b/lib/python3.12/site-packages/datasets/formatting/np_formatter.py new file mode 100644 index 0000000000000000000000000000000000000000..c12ecd4f3866cab1d9740d962ebab8ad762f8d53 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/formatting/np_formatter.py @@ -0,0 +1,112 @@ +# Copyright 2020 The HuggingFace Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from collections.abc import Mapping + +import numpy as np +import pyarrow as pa + +from .. import config +from ..utils.py_utils import map_nested +from .formatting import TensorFormatter + + +class NumpyFormatter(TensorFormatter[Mapping, np.ndarray, Mapping]): + def __init__(self, features=None, token_per_repo_id=None, **np_array_kwargs): + super().__init__(features=features, token_per_repo_id=token_per_repo_id) + self.np_array_kwargs = np_array_kwargs + + def _consolidate(self, column): + if isinstance(column, list): + if column and all( + isinstance(x, np.ndarray) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column + ): + return np.stack(column) + else: + # don't use np.array(column, dtype=object) + # since it fails in certain cases + # see https://stackoverflow.com/q/51005699 + out = np.empty(len(column), dtype=object) + out[:] = column + return out + return column + + def _tensorize(self, value): + if isinstance(value, (str, bytes, type(None))): + return value + elif isinstance(value, (np.character, np.ndarray)) and np.issubdtype(value.dtype, np.character): + return value + elif isinstance(value, np.number): + return value + + default_dtype = {} + + if isinstance(value, np.ndarray) and np.issubdtype(value.dtype, np.integer): + default_dtype = {"dtype": np.int64} + elif isinstance(value, np.ndarray) and np.issubdtype(value.dtype, np.floating): + default_dtype = {"dtype": np.float32} + + if config.PIL_AVAILABLE and "PIL" in sys.modules: + import PIL.Image + + if isinstance(value, PIL.Image.Image): + return np.asarray(value, **self.np_array_kwargs) + if config.TORCHVISION_AVAILABLE and "torchvision" in sys.modules: + from torchvision.io import VideoReader + + if isinstance(value, VideoReader): + return value # TODO(QL): set output to np arrays ? + + return np.asarray(value, **{**default_dtype, **self.np_array_kwargs}) + + def _recursive_tensorize(self, data_struct): + # support for torch, tf, jax etc. + if config.TORCH_AVAILABLE and "torch" in sys.modules: + import torch + + if isinstance(data_struct, torch.Tensor): + return self._tensorize(data_struct.detach().cpu().numpy()[()]) + if hasattr(data_struct, "__array__") and not isinstance(data_struct, (np.ndarray, np.character, np.number)): + data_struct = data_struct.__array__() + # support for nested types like struct of list of struct + if isinstance(data_struct, np.ndarray): + if data_struct.dtype == object: + return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct]) + if isinstance(data_struct, (list, tuple)): + return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct]) + return self._tensorize(data_struct) + + def recursive_tensorize(self, data_struct: dict): + return map_nested(self._recursive_tensorize, data_struct, map_list=False) + + def format_row(self, pa_table: pa.Table) -> Mapping: + row = self.numpy_arrow_extractor().extract_row(pa_table) + row = self.python_features_decoder.decode_row(row) + return self.recursive_tensorize(row) + + def format_column(self, pa_table: pa.Table) -> np.ndarray: + column = self.numpy_arrow_extractor().extract_column(pa_table) + column = self.python_features_decoder.decode_column(column, pa_table.column_names[0]) + column = self.recursive_tensorize(column) + column = self._consolidate(column) + return column + + def format_batch(self, pa_table: pa.Table) -> Mapping: + batch = self.numpy_arrow_extractor().extract_batch(pa_table) + batch = self.python_features_decoder.decode_batch(batch) + batch = self.recursive_tensorize(batch) + for column_name in batch: + batch[column_name] = self._consolidate(batch[column_name]) + return batch diff --git a/lib/python3.12/site-packages/datasets/formatting/polars_formatter.py b/lib/python3.12/site-packages/datasets/formatting/polars_formatter.py new file mode 100644 index 0000000000000000000000000000000000000000..7ea2f783aec38ac6bbc10d12aff2488655940f3f --- /dev/null +++ b/lib/python3.12/site-packages/datasets/formatting/polars_formatter.py @@ -0,0 +1,124 @@ +# Copyright 2020 The HuggingFace Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from functools import partial +from typing import TYPE_CHECKING, Optional + +import pyarrow as pa + +from .. import config +from ..features import Features +from ..features.features import decode_nested_example +from ..utils.py_utils import no_op_if_value_is_null +from .formatting import BaseArrowExtractor, TableFormatter + + +if TYPE_CHECKING: + import polars as pl + + +class PolarsArrowExtractor(BaseArrowExtractor["pl.DataFrame", "pl.Series", "pl.DataFrame"]): + def extract_row(self, pa_table: pa.Table) -> "pl.DataFrame": + if config.POLARS_AVAILABLE: + if "polars" not in sys.modules: + import polars + else: + polars = sys.modules["polars"] + + return polars.from_arrow(pa_table.slice(length=1)) + else: + raise ValueError("Polars needs to be installed to be able to return Polars dataframes.") + + def extract_column(self, pa_table: pa.Table) -> "pl.Series": + if config.POLARS_AVAILABLE: + if "polars" not in sys.modules: + import polars + else: + polars = sys.modules["polars"] + + return polars.from_arrow(pa_table.select([0]))[pa_table.column_names[0]] + else: + raise ValueError("Polars needs to be installed to be able to return Polars dataframes.") + + def extract_batch(self, pa_table: pa.Table) -> "pl.DataFrame": + if config.POLARS_AVAILABLE: + if "polars" not in sys.modules: + import polars + else: + polars = sys.modules["polars"] + + return polars.from_arrow(pa_table) + else: + raise ValueError("Polars needs to be installed to be able to return Polars dataframes.") + + +class PolarsFeaturesDecoder: + def __init__(self, features: Optional[Features]): + self.features = features + import polars as pl # noqa: F401 - import pl at initialization + + def decode_row(self, row: "pl.DataFrame") -> "pl.DataFrame": + decode = ( + { + column_name: no_op_if_value_is_null(partial(decode_nested_example, feature)) + for column_name, feature in self.features.items() + if self.features._column_requires_decoding[column_name] + } + if self.features + else {} + ) + if decode: + row[list(decode.keys())] = row.map_rows(decode) + return row + + def decode_column(self, column: "pl.Series", column_name: str) -> "pl.Series": + decode = ( + no_op_if_value_is_null(partial(decode_nested_example, self.features[column_name])) + if self.features and column_name in self.features and self.features._column_requires_decoding[column_name] + else None + ) + if decode: + column = column.map_elements(decode) + return column + + def decode_batch(self, batch: "pl.DataFrame") -> "pl.DataFrame": + return self.decode_row(batch) + + +class PolarsFormatter(TableFormatter["pl.DataFrame", "pl.Series", "pl.DataFrame"]): + table_type = "polars dataframe" + column_type = "polars series" + + def __init__(self, features=None, **np_array_kwargs): + super().__init__(features=features) + self.np_array_kwargs = np_array_kwargs + self.polars_arrow_extractor = PolarsArrowExtractor + self.polars_features_decoder = PolarsFeaturesDecoder(features) + import polars as pl # noqa: F401 - import pl at initialization + + def format_row(self, pa_table: pa.Table) -> "pl.DataFrame": + row = self.polars_arrow_extractor().extract_row(pa_table) + row = self.polars_features_decoder.decode_row(row) + return row + + def format_column(self, pa_table: pa.Table) -> "pl.Series": + column = self.polars_arrow_extractor().extract_column(pa_table) + column = self.polars_features_decoder.decode_column(column, pa_table.column_names[0]) + return column + + def format_batch(self, pa_table: pa.Table) -> "pl.DataFrame": + row = self.polars_arrow_extractor().extract_batch(pa_table) + row = self.polars_features_decoder.decode_batch(row) + return row diff --git a/lib/python3.12/site-packages/datasets/formatting/tf_formatter.py b/lib/python3.12/site-packages/datasets/formatting/tf_formatter.py new file mode 100644 index 0000000000000000000000000000000000000000..964bc4b5589a25fe7690706c780c2b4c07e94a5a --- /dev/null +++ b/lib/python3.12/site-packages/datasets/formatting/tf_formatter.py @@ -0,0 +1,121 @@ +# Copyright 2020 The HuggingFace Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +import sys +from collections.abc import Mapping +from typing import TYPE_CHECKING + +import numpy as np +import pyarrow as pa + +from .. import config +from ..utils.py_utils import map_nested +from .formatting import TensorFormatter + + +if TYPE_CHECKING: + import tensorflow as tf + + +class TFFormatter(TensorFormatter[Mapping, "tf.Tensor", Mapping]): + def __init__(self, features=None, token_per_repo_id=None, **tf_tensor_kwargs): + super().__init__(features=features, token_per_repo_id=token_per_repo_id) + self.tf_tensor_kwargs = tf_tensor_kwargs + import tensorflow as tf # noqa: F401 - import tf at initialization + + def _consolidate(self, column): + import tensorflow as tf + + if isinstance(column, list) and column: + if all( + isinstance(x, tf.Tensor) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column + ): + return tf.stack(column) + elif all( + isinstance(x, (tf.Tensor, tf.RaggedTensor)) and x.ndim == 1 and x.dtype == column[0].dtype + for x in column + ): + # only rag 1-D tensors, otherwise some dimensions become ragged even though they were consolidated + return tf.ragged.stack(column) + + return column + + def _tensorize(self, value): + import tensorflow as tf + + if value is None: + return value + + default_dtype = {} + + if isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.integer): + default_dtype = {"dtype": tf.int64} + elif isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.floating): + default_dtype = {"dtype": tf.float32} + + if config.PIL_AVAILABLE and "PIL" in sys.modules: + import PIL.Image + + if isinstance(value, PIL.Image.Image): + value = np.asarray(value) + if config.TORCHVISION_AVAILABLE and "torchvision" in sys.modules: + from torchvision.io import VideoReader + + if isinstance(value, VideoReader): + return value # TODO(QL): set output to tf tensors ? + + return tf.convert_to_tensor(value, **{**default_dtype, **self.tf_tensor_kwargs}) + + def _recursive_tensorize(self, data_struct): + import tensorflow as tf + + # support for torch, tf, jax etc. + if config.TORCH_AVAILABLE and "torch" in sys.modules: + import torch + + if isinstance(data_struct, torch.Tensor): + return self._tensorize(data_struct.detach().cpu().numpy()[()]) + if hasattr(data_struct, "__array__") and not isinstance(data_struct, tf.Tensor): + data_struct = data_struct.__array__() + # support for nested types like struct of list of struct + if isinstance(data_struct, np.ndarray): + if data_struct.dtype == object: # tf tensors cannot be instantied from an array of objects + return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct]) + elif isinstance(data_struct, (list, tuple)): + return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct]) + return self._tensorize(data_struct) + + def recursive_tensorize(self, data_struct: dict): + return map_nested(self._recursive_tensorize, data_struct, map_list=False) + + def format_row(self, pa_table: pa.Table) -> Mapping: + row = self.numpy_arrow_extractor().extract_row(pa_table) + row = self.python_features_decoder.decode_row(row) + return self.recursive_tensorize(row) + + def format_column(self, pa_table: pa.Table) -> "tf.Tensor": + column = self.numpy_arrow_extractor().extract_column(pa_table) + column = self.python_features_decoder.decode_column(column, pa_table.column_names[0]) + column = self.recursive_tensorize(column) + column = self._consolidate(column) + return column + + def format_batch(self, pa_table: pa.Table) -> Mapping: + batch = self.numpy_arrow_extractor().extract_batch(pa_table) + batch = self.python_features_decoder.decode_batch(batch) + batch = self.recursive_tensorize(batch) + for column_name in batch: + batch[column_name] = self._consolidate(batch[column_name]) + return batch diff --git a/lib/python3.12/site-packages/datasets/formatting/torch_formatter.py b/lib/python3.12/site-packages/datasets/formatting/torch_formatter.py new file mode 100644 index 0000000000000000000000000000000000000000..cfc3fd9abe870a5354ef6d2f9020ca255be721a8 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/formatting/torch_formatter.py @@ -0,0 +1,122 @@ +# Copyright 2020 The HuggingFace Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +import sys +from collections.abc import Mapping +from typing import TYPE_CHECKING + +import numpy as np +import pyarrow as pa + +from .. import config +from ..utils.py_utils import map_nested +from .formatting import TensorFormatter + + +if TYPE_CHECKING: + import torch + + +class TorchFormatter(TensorFormatter[Mapping, "torch.Tensor", Mapping]): + def __init__(self, features=None, token_per_repo_id=None, **torch_tensor_kwargs): + super().__init__(features=features, token_per_repo_id=token_per_repo_id) + self.torch_tensor_kwargs = torch_tensor_kwargs + import torch # noqa import torch at initialization + + def _consolidate(self, column): + import torch + + if isinstance(column, list) and column: + if all( + isinstance(x, torch.Tensor) and x.shape == column[0].shape and x.dtype == column[0].dtype + for x in column + ): + return torch.stack(column) + return column + + def _tensorize(self, value): + import torch + + if isinstance(value, (str, bytes, type(None))): + return value + elif isinstance(value, (np.character, np.ndarray)) and np.issubdtype(value.dtype, np.character): + return value.tolist() + + default_dtype = {} + + if isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.integer): + default_dtype = {"dtype": torch.int64} + + # Convert dtype to np.int64 if it's either np.uint16 or np.uint32 to ensure compatibility. + # np.uint64 is excluded from this conversion as there is no compatible PyTorch dtype that can handle it without loss. + if value.dtype in [np.uint16, np.uint32]: + value = value.astype(np.int64) + + elif isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.floating): + default_dtype = {"dtype": torch.float32} + + if config.PIL_AVAILABLE and "PIL" in sys.modules: + import PIL.Image + + if isinstance(value, PIL.Image.Image): + value = np.asarray(value) + if value.ndim == 2: + value = value[:, :, np.newaxis] + + value = value.transpose((2, 0, 1)) + if config.TORCHVISION_AVAILABLE and "torchvision" in sys.modules: + from torchvision.io import VideoReader + + if isinstance(value, VideoReader): + return value # TODO(QL): set output to torch tensors ? + + return torch.tensor(value, **{**default_dtype, **self.torch_tensor_kwargs}) + + def _recursive_tensorize(self, data_struct): + import torch + + # support for torch, tf, jax etc. + if hasattr(data_struct, "__array__") and not isinstance(data_struct, torch.Tensor): + data_struct = data_struct.__array__() + # support for nested types like struct of list of struct + if isinstance(data_struct, np.ndarray): + if data_struct.dtype == object: # torch tensors cannot be instantied from an array of objects + return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct]) + elif isinstance(data_struct, (list, tuple)): + return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct]) + return self._tensorize(data_struct) + + def recursive_tensorize(self, data_struct: dict): + return map_nested(self._recursive_tensorize, data_struct, map_list=False) + + def format_row(self, pa_table: pa.Table) -> Mapping: + row = self.numpy_arrow_extractor().extract_row(pa_table) + row = self.python_features_decoder.decode_row(row) + return self.recursive_tensorize(row) + + def format_column(self, pa_table: pa.Table) -> "torch.Tensor": + column = self.numpy_arrow_extractor().extract_column(pa_table) + column = self.python_features_decoder.decode_column(column, pa_table.column_names[0]) + column = self.recursive_tensorize(column) + column = self._consolidate(column) + return column + + def format_batch(self, pa_table: pa.Table) -> Mapping: + batch = self.numpy_arrow_extractor().extract_batch(pa_table) + batch = self.python_features_decoder.decode_batch(batch) + batch = self.recursive_tensorize(batch) + for column_name in batch: + batch[column_name] = self._consolidate(batch[column_name]) + return batch diff --git a/lib/python3.12/site-packages/datasets/io/__init__.py b/lib/python3.12/site-packages/datasets/io/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/datasets/io/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/io/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ff095749d4cfb0b34bce69509ed7f7fab92e85df Binary files /dev/null and b/lib/python3.12/site-packages/datasets/io/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/io/__pycache__/abc.cpython-312.pyc b/lib/python3.12/site-packages/datasets/io/__pycache__/abc.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..be48047183f3312a4301516ea5d27506aaa5c7cb Binary files /dev/null and b/lib/python3.12/site-packages/datasets/io/__pycache__/abc.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/io/__pycache__/csv.cpython-312.pyc b/lib/python3.12/site-packages/datasets/io/__pycache__/csv.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4c2a8e4697c2f7f0ee52ae07d1176edeb8d94fc0 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/io/__pycache__/csv.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/io/__pycache__/generator.cpython-312.pyc b/lib/python3.12/site-packages/datasets/io/__pycache__/generator.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..df70b666c57de7f2c7d41122b6c8c6599987df55 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/io/__pycache__/generator.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/io/__pycache__/json.cpython-312.pyc b/lib/python3.12/site-packages/datasets/io/__pycache__/json.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b2fa1112e2fd8b7a5fda7cba34f152478b18df3b Binary files /dev/null and b/lib/python3.12/site-packages/datasets/io/__pycache__/json.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/io/__pycache__/parquet.cpython-312.pyc b/lib/python3.12/site-packages/datasets/io/__pycache__/parquet.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..805fea22c948c0e420c349ebd8b5c2c44a3fb713 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/io/__pycache__/parquet.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/io/__pycache__/spark.cpython-312.pyc b/lib/python3.12/site-packages/datasets/io/__pycache__/spark.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..64dc0786814ca2230580e2abb35428d31e479661 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/io/__pycache__/spark.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/io/__pycache__/sql.cpython-312.pyc b/lib/python3.12/site-packages/datasets/io/__pycache__/sql.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..66a4a8e59c8bd316f7d10d05a42ad1d8d727330a Binary files /dev/null and b/lib/python3.12/site-packages/datasets/io/__pycache__/sql.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/io/__pycache__/text.cpython-312.pyc b/lib/python3.12/site-packages/datasets/io/__pycache__/text.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d9af00379e973ab9358d8a4d2940d2af5d0d6118 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/io/__pycache__/text.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/io/abc.py b/lib/python3.12/site-packages/datasets/io/abc.py new file mode 100644 index 0000000000000000000000000000000000000000..a1913cc20e3fd748ef912e2fb3d7c1e18f16ac8c --- /dev/null +++ b/lib/python3.12/site-packages/datasets/io/abc.py @@ -0,0 +1,53 @@ +from abc import ABC, abstractmethod +from typing import Optional, Union + +from .. import Dataset, DatasetDict, Features, IterableDataset, IterableDatasetDict, NamedSplit +from ..utils.typing import NestedDataStructureLike, PathLike + + +class AbstractDatasetReader(ABC): + def __init__( + self, + path_or_paths: Optional[NestedDataStructureLike[PathLike]] = None, + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + streaming: bool = False, + num_proc: Optional[int] = None, + **kwargs, + ): + self.path_or_paths = path_or_paths + self.split = split if split or isinstance(path_or_paths, dict) else "train" + self.features = features + self.cache_dir = cache_dir + self.keep_in_memory = keep_in_memory + self.streaming = streaming + self.num_proc = num_proc + self.kwargs = kwargs + + @abstractmethod + def read(self) -> Union[Dataset, DatasetDict, IterableDataset, IterableDatasetDict]: + pass + + +class AbstractDatasetInputStream(ABC): + def __init__( + self, + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + streaming: bool = False, + num_proc: Optional[int] = None, + **kwargs, + ): + self.features = features + self.cache_dir = cache_dir + self.keep_in_memory = keep_in_memory + self.streaming = streaming + self.num_proc = num_proc + self.kwargs = kwargs + + @abstractmethod + def read(self) -> Union[Dataset, IterableDataset]: + pass diff --git a/lib/python3.12/site-packages/datasets/io/csv.py b/lib/python3.12/site-packages/datasets/io/csv.py new file mode 100644 index 0000000000000000000000000000000000000000..4ac2ea1135b579ae0f38f7dafc73ae5c2db1a6b6 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/io/csv.py @@ -0,0 +1,145 @@ +import multiprocessing +import os +from typing import BinaryIO, Optional, Union + +import fsspec + +from .. import Dataset, Features, NamedSplit, config +from ..formatting import query_table +from ..packaged_modules.csv.csv import Csv +from ..utils import tqdm as hf_tqdm +from ..utils.typing import NestedDataStructureLike, PathLike +from .abc import AbstractDatasetReader + + +class CsvDatasetReader(AbstractDatasetReader): + def __init__( + self, + path_or_paths: NestedDataStructureLike[PathLike], + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + streaming: bool = False, + num_proc: Optional[int] = None, + **kwargs, + ): + super().__init__( + path_or_paths, + split=split, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + streaming=streaming, + num_proc=num_proc, + **kwargs, + ) + path_or_paths = path_or_paths if isinstance(path_or_paths, dict) else {self.split: path_or_paths} + self.builder = Csv( + cache_dir=cache_dir, + data_files=path_or_paths, + features=features, + **kwargs, + ) + + def read(self): + # Build iterable dataset + if self.streaming: + dataset = self.builder.as_streaming_dataset(split=self.split) + # Build regular (map-style) dataset + else: + download_config = None + download_mode = None + verification_mode = None + base_path = None + + self.builder.download_and_prepare( + download_config=download_config, + download_mode=download_mode, + verification_mode=verification_mode, + base_path=base_path, + num_proc=self.num_proc, + ) + dataset = self.builder.as_dataset( + split=self.split, verification_mode=verification_mode, in_memory=self.keep_in_memory + ) + return dataset + + +class CsvDatasetWriter: + def __init__( + self, + dataset: Dataset, + path_or_buf: Union[PathLike, BinaryIO], + batch_size: Optional[int] = None, + num_proc: Optional[int] = None, + storage_options: Optional[dict] = None, + **to_csv_kwargs, + ): + if num_proc is not None and num_proc <= 0: + raise ValueError(f"num_proc {num_proc} must be an integer > 0.") + + self.dataset = dataset + self.path_or_buf = path_or_buf + self.batch_size = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE + self.num_proc = num_proc + self.encoding = "utf-8" + self.storage_options = storage_options or {} + self.to_csv_kwargs = to_csv_kwargs + + def write(self) -> int: + _ = self.to_csv_kwargs.pop("path_or_buf", None) + header = self.to_csv_kwargs.pop("header", True) + index = self.to_csv_kwargs.pop("index", False) + + if isinstance(self.path_or_buf, (str, bytes, os.PathLike)): + with fsspec.open(self.path_or_buf, "wb", **(self.storage_options or {})) as buffer: + written = self._write(file_obj=buffer, header=header, index=index, **self.to_csv_kwargs) + else: + written = self._write(file_obj=self.path_or_buf, header=header, index=index, **self.to_csv_kwargs) + return written + + def _batch_csv(self, args): + offset, header, index, to_csv_kwargs = args + + batch = query_table( + table=self.dataset.data, + key=slice(offset, offset + self.batch_size), + indices=self.dataset._indices, + ) + csv_str = batch.to_pandas().to_csv( + path_or_buf=None, header=header if (offset == 0) else False, index=index, **to_csv_kwargs + ) + return csv_str.encode(self.encoding) + + def _write(self, file_obj: BinaryIO, header, index, **to_csv_kwargs) -> int: + """Writes the pyarrow table as CSV to a binary file handle. + + Caller is responsible for opening and closing the handle. + """ + written = 0 + + if self.num_proc is None or self.num_proc == 1: + for offset in hf_tqdm( + range(0, len(self.dataset), self.batch_size), + unit="ba", + desc="Creating CSV from Arrow format", + ): + csv_str = self._batch_csv((offset, header, index, to_csv_kwargs)) + written += file_obj.write(csv_str) + + else: + num_rows, batch_size = len(self.dataset), self.batch_size + with multiprocessing.Pool(self.num_proc) as pool: + for csv_str in hf_tqdm( + pool.imap( + self._batch_csv, + [(offset, header, index, to_csv_kwargs) for offset in range(0, num_rows, batch_size)], + ), + total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size, + unit="ba", + desc="Creating CSV from Arrow format", + ): + written += file_obj.write(csv_str) + + return written diff --git a/lib/python3.12/site-packages/datasets/io/generator.py b/lib/python3.12/site-packages/datasets/io/generator.py new file mode 100644 index 0000000000000000000000000000000000000000..b10609cac23493ec752080bf19ac166aaf5c63a9 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/io/generator.py @@ -0,0 +1,59 @@ +from typing import Callable, Optional + +from .. import Features, NamedSplit, Split +from ..packaged_modules.generator.generator import Generator +from .abc import AbstractDatasetInputStream + + +class GeneratorDatasetInputStream(AbstractDatasetInputStream): + def __init__( + self, + generator: Callable, + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + streaming: bool = False, + gen_kwargs: Optional[dict] = None, + num_proc: Optional[int] = None, + split: NamedSplit = Split.TRAIN, + **kwargs, + ): + super().__init__( + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + streaming=streaming, + num_proc=num_proc, + **kwargs, + ) + self.builder = Generator( + cache_dir=cache_dir, + features=features, + generator=generator, + gen_kwargs=gen_kwargs, + split=split, + **kwargs, + ) + + def read(self): + # Build iterable dataset + if self.streaming: + dataset = self.builder.as_streaming_dataset(split=self.builder.config.split) + # Build regular (map-style) dataset + else: + download_config = None + download_mode = None + verification_mode = None + base_path = None + + self.builder.download_and_prepare( + download_config=download_config, + download_mode=download_mode, + verification_mode=verification_mode, + base_path=base_path, + num_proc=self.num_proc, + ) + dataset = self.builder.as_dataset( + split=self.builder.config.split, verification_mode=verification_mode, in_memory=self.keep_in_memory + ) + return dataset diff --git a/lib/python3.12/site-packages/datasets/io/json.py b/lib/python3.12/site-packages/datasets/io/json.py new file mode 100644 index 0000000000000000000000000000000000000000..41abfa518ccb7b46c446b98eb249f4c95ed9a731 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/io/json.py @@ -0,0 +1,174 @@ +import multiprocessing +import os +from typing import BinaryIO, Optional, Union + +import fsspec + +from .. import Dataset, Features, NamedSplit, config +from ..formatting import query_table +from ..packaged_modules.json.json import Json +from ..utils import tqdm as hf_tqdm +from ..utils.typing import NestedDataStructureLike, PathLike +from .abc import AbstractDatasetReader + + +class JsonDatasetReader(AbstractDatasetReader): + def __init__( + self, + path_or_paths: NestedDataStructureLike[PathLike], + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + streaming: bool = False, + field: Optional[str] = None, + num_proc: Optional[int] = None, + **kwargs, + ): + super().__init__( + path_or_paths, + split=split, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + streaming=streaming, + num_proc=num_proc, + **kwargs, + ) + self.field = field + path_or_paths = path_or_paths if isinstance(path_or_paths, dict) else {self.split: path_or_paths} + self.builder = Json( + cache_dir=cache_dir, + data_files=path_or_paths, + features=features, + field=field, + **kwargs, + ) + + def read(self): + # Build iterable dataset + if self.streaming: + dataset = self.builder.as_streaming_dataset(split=self.split) + # Build regular (map-style) dataset + else: + download_config = None + download_mode = None + verification_mode = None + base_path = None + + self.builder.download_and_prepare( + download_config=download_config, + download_mode=download_mode, + verification_mode=verification_mode, + base_path=base_path, + num_proc=self.num_proc, + ) + dataset = self.builder.as_dataset( + split=self.split, verification_mode=verification_mode, in_memory=self.keep_in_memory + ) + return dataset + + +class JsonDatasetWriter: + def __init__( + self, + dataset: Dataset, + path_or_buf: Union[PathLike, BinaryIO], + batch_size: Optional[int] = None, + num_proc: Optional[int] = None, + storage_options: Optional[dict] = None, + **to_json_kwargs, + ): + if num_proc is not None and num_proc <= 0: + raise ValueError(f"num_proc {num_proc} must be an integer > 0.") + + self.dataset = dataset + self.path_or_buf = path_or_buf + self.batch_size = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE + self.num_proc = num_proc + self.encoding = "utf-8" + self.storage_options = storage_options or {} + self.to_json_kwargs = to_json_kwargs + + def write(self) -> int: + _ = self.to_json_kwargs.pop("path_or_buf", None) + orient = self.to_json_kwargs.pop("orient", "records") + lines = self.to_json_kwargs.pop("lines", True if orient == "records" else False) + if "index" not in self.to_json_kwargs and orient in ["split", "table"]: + self.to_json_kwargs["index"] = False + + # Determine the default compression value based on self.path_or_buf type + default_compression = "infer" if isinstance(self.path_or_buf, (str, bytes, os.PathLike)) else None + compression = self.to_json_kwargs.pop("compression", default_compression) + + if compression not in [None, "infer", "gzip", "bz2", "xz"]: + raise NotImplementedError(f"`datasets` currently does not support {compression} compression") + + if not lines and self.batch_size < self.dataset.num_rows: + raise NotImplementedError( + "Output JSON will not be formatted correctly when lines = False and batch_size < number of rows in the dataset. Use pandas.DataFrame.to_json() instead." + ) + + if isinstance(self.path_or_buf, (str, bytes, os.PathLike)): + with fsspec.open( + self.path_or_buf, "wb", compression=compression, **(self.storage_options or {}) + ) as buffer: + written = self._write(file_obj=buffer, orient=orient, lines=lines, **self.to_json_kwargs) + else: + if compression: + raise NotImplementedError( + f"The compression parameter is not supported when writing to a buffer, but compression={compression}" + " was passed. Please provide a local path instead." + ) + written = self._write(file_obj=self.path_or_buf, orient=orient, lines=lines, **self.to_json_kwargs) + return written + + def _batch_json(self, args): + offset, orient, lines, to_json_kwargs = args + + batch = query_table( + table=self.dataset.data, + key=slice(offset, offset + self.batch_size), + indices=self.dataset._indices, + ) + json_str = batch.to_pandas().to_json(path_or_buf=None, orient=orient, lines=lines, **to_json_kwargs) + if not json_str.endswith("\n"): + json_str += "\n" + return json_str.encode(self.encoding) + + def _write( + self, + file_obj: BinaryIO, + orient, + lines, + **to_json_kwargs, + ) -> int: + """Writes the pyarrow table as JSON lines to a binary file handle. + + Caller is responsible for opening and closing the handle. + """ + written = 0 + + if self.num_proc is None or self.num_proc == 1: + for offset in hf_tqdm( + range(0, len(self.dataset), self.batch_size), + unit="ba", + desc="Creating json from Arrow format", + ): + json_str = self._batch_json((offset, orient, lines, to_json_kwargs)) + written += file_obj.write(json_str) + else: + num_rows, batch_size = len(self.dataset), self.batch_size + with multiprocessing.Pool(self.num_proc) as pool: + for json_str in hf_tqdm( + pool.imap( + self._batch_json, + [(offset, orient, lines, to_json_kwargs) for offset in range(0, num_rows, batch_size)], + ), + total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size, + unit="ba", + desc="Creating json from Arrow format", + ): + written += file_obj.write(json_str) + + return written diff --git a/lib/python3.12/site-packages/datasets/io/parquet.py b/lib/python3.12/site-packages/datasets/io/parquet.py new file mode 100644 index 0000000000000000000000000000000000000000..d34f51102046a39a9dd0bba3eb3eb982e223d187 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/io/parquet.py @@ -0,0 +1,122 @@ +import os +from typing import BinaryIO, Optional, Union + +import fsspec +import pyarrow.parquet as pq + +from .. import Dataset, Features, NamedSplit, config +from ..arrow_writer import get_writer_batch_size +from ..formatting import query_table +from ..packaged_modules import _PACKAGED_DATASETS_MODULES +from ..packaged_modules.parquet.parquet import Parquet +from ..utils import tqdm as hf_tqdm +from ..utils.typing import NestedDataStructureLike, PathLike +from .abc import AbstractDatasetReader + + +class ParquetDatasetReader(AbstractDatasetReader): + def __init__( + self, + path_or_paths: NestedDataStructureLike[PathLike], + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + streaming: bool = False, + num_proc: Optional[int] = None, + **kwargs, + ): + super().__init__( + path_or_paths, + split=split, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + streaming=streaming, + num_proc=num_proc, + **kwargs, + ) + path_or_paths = path_or_paths if isinstance(path_or_paths, dict) else {self.split: path_or_paths} + hash = _PACKAGED_DATASETS_MODULES["parquet"][1] + self.builder = Parquet( + cache_dir=cache_dir, + data_files=path_or_paths, + features=features, + hash=hash, + **kwargs, + ) + + def read(self): + # Build iterable dataset + if self.streaming: + dataset = self.builder.as_streaming_dataset(split=self.split) + # Build regular (map-style) dataset + else: + download_config = None + download_mode = None + verification_mode = None + base_path = None + + self.builder.download_and_prepare( + download_config=download_config, + download_mode=download_mode, + verification_mode=verification_mode, + base_path=base_path, + num_proc=self.num_proc, + ) + dataset = self.builder.as_dataset( + split=self.split, verification_mode=verification_mode, in_memory=self.keep_in_memory + ) + return dataset + + +class ParquetDatasetWriter: + def __init__( + self, + dataset: Dataset, + path_or_buf: Union[PathLike, BinaryIO], + batch_size: Optional[int] = None, + storage_options: Optional[dict] = None, + **parquet_writer_kwargs, + ): + self.dataset = dataset + self.path_or_buf = path_or_buf + self.batch_size = batch_size or get_writer_batch_size(dataset.features) + self.storage_options = storage_options or {} + self.parquet_writer_kwargs = parquet_writer_kwargs + + def write(self) -> int: + batch_size = self.batch_size if self.batch_size else config.DEFAULT_MAX_BATCH_SIZE + + if isinstance(self.path_or_buf, (str, bytes, os.PathLike)): + with fsspec.open(self.path_or_buf, "wb", **(self.storage_options or {})) as buffer: + written = self._write(file_obj=buffer, batch_size=batch_size, **self.parquet_writer_kwargs) + else: + written = self._write(file_obj=self.path_or_buf, batch_size=batch_size, **self.parquet_writer_kwargs) + return written + + def _write(self, file_obj: BinaryIO, batch_size: int, **parquet_writer_kwargs) -> int: + """Writes the pyarrow table as Parquet to a binary file handle. + + Caller is responsible for opening and closing the handle. + """ + written = 0 + _ = parquet_writer_kwargs.pop("path_or_buf", None) + schema = self.dataset.features.arrow_schema + + writer = pq.ParquetWriter(file_obj, schema=schema, **parquet_writer_kwargs) + + for offset in hf_tqdm( + range(0, len(self.dataset), batch_size), + unit="ba", + desc="Creating parquet from Arrow format", + ): + batch = query_table( + table=self.dataset._data, + key=slice(offset, offset + batch_size), + indices=self.dataset._indices, + ) + writer.write_table(batch) + written += batch.nbytes + writer.close() + return written diff --git a/lib/python3.12/site-packages/datasets/io/spark.py b/lib/python3.12/site-packages/datasets/io/spark.py new file mode 100644 index 0000000000000000000000000000000000000000..7562ba1fb5f77ed8f82374e3021fcb3a93b1da8d --- /dev/null +++ b/lib/python3.12/site-packages/datasets/io/spark.py @@ -0,0 +1,57 @@ +from typing import Optional + +import pyspark + +from .. import Features, NamedSplit +from ..download import DownloadMode +from ..packaged_modules.spark.spark import Spark +from .abc import AbstractDatasetReader + + +class SparkDatasetReader(AbstractDatasetReader): + """A dataset reader that reads from a Spark DataFrame. + + When caching, cache materialization is parallelized over Spark; an NFS that is accessible to the driver must be + provided. Streaming is not currently supported. + """ + + def __init__( + self, + df: pyspark.sql.DataFrame, + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + streaming: bool = True, + cache_dir: str = None, + keep_in_memory: bool = False, + working_dir: str = None, + load_from_cache_file: bool = True, + file_format: str = "arrow", + **kwargs, + ): + super().__init__( + split=split, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + streaming=streaming, + **kwargs, + ) + self._load_from_cache_file = load_from_cache_file + self._file_format = file_format + self.builder = Spark( + df=df, + features=features, + cache_dir=cache_dir, + working_dir=working_dir, + **kwargs, + ) + + def read(self): + if self.streaming: + return self.builder.as_streaming_dataset(split=self.split) + download_mode = None if self._load_from_cache_file else DownloadMode.FORCE_REDOWNLOAD + self.builder.download_and_prepare( + download_mode=download_mode, + file_format=self._file_format, + ) + return self.builder.as_dataset(split=self.split) diff --git a/lib/python3.12/site-packages/datasets/io/sql.py b/lib/python3.12/site-packages/datasets/io/sql.py new file mode 100644 index 0000000000000000000000000000000000000000..2331e3e6407fa81cd26569306ff2c7952dad4920 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/io/sql.py @@ -0,0 +1,124 @@ +import multiprocessing +from typing import TYPE_CHECKING, Optional, Union + +from .. import Dataset, Features, config +from ..formatting import query_table +from ..packaged_modules.sql.sql import Sql +from ..utils import tqdm as hf_tqdm +from .abc import AbstractDatasetInputStream + + +if TYPE_CHECKING: + import sqlite3 + + import sqlalchemy + + +class SqlDatasetReader(AbstractDatasetInputStream): + def __init__( + self, + sql: Union[str, "sqlalchemy.sql.Selectable"], + con: Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"], + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + **kwargs, + ): + super().__init__(features=features, cache_dir=cache_dir, keep_in_memory=keep_in_memory, **kwargs) + self.builder = Sql( + cache_dir=cache_dir, + features=features, + sql=sql, + con=con, + **kwargs, + ) + + def read(self): + download_config = None + download_mode = None + verification_mode = None + base_path = None + + self.builder.download_and_prepare( + download_config=download_config, + download_mode=download_mode, + verification_mode=verification_mode, + base_path=base_path, + ) + + # Build dataset for splits + dataset = self.builder.as_dataset( + split="train", verification_mode=verification_mode, in_memory=self.keep_in_memory + ) + return dataset + + +class SqlDatasetWriter: + def __init__( + self, + dataset: Dataset, + name: str, + con: Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"], + batch_size: Optional[int] = None, + num_proc: Optional[int] = None, + **to_sql_kwargs, + ): + if num_proc is not None and num_proc <= 0: + raise ValueError(f"num_proc {num_proc} must be an integer > 0.") + + self.dataset = dataset + self.name = name + self.con = con + self.batch_size = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE + self.num_proc = num_proc + self.to_sql_kwargs = to_sql_kwargs + + def write(self) -> int: + _ = self.to_sql_kwargs.pop("sql", None) + _ = self.to_sql_kwargs.pop("con", None) + index = self.to_sql_kwargs.pop("index", False) + + written = self._write(index=index, **self.to_sql_kwargs) + return written + + def _batch_sql(self, args): + offset, index, to_sql_kwargs = args + to_sql_kwargs = {**to_sql_kwargs, "if_exists": "append"} if offset > 0 else to_sql_kwargs + batch = query_table( + table=self.dataset.data, + key=slice(offset, offset + self.batch_size), + indices=self.dataset._indices, + ) + df = batch.to_pandas() + num_rows = df.to_sql(self.name, self.con, index=index, **to_sql_kwargs) + return num_rows or len(df) + + def _write(self, index, **to_sql_kwargs) -> int: + """Writes the pyarrow table as SQL to a database. + + Caller is responsible for opening and closing the SQL connection. + """ + written = 0 + + if self.num_proc is None or self.num_proc == 1: + for offset in hf_tqdm( + range(0, len(self.dataset), self.batch_size), + unit="ba", + desc="Creating SQL from Arrow format", + ): + written += self._batch_sql((offset, index, to_sql_kwargs)) + else: + num_rows, batch_size = len(self.dataset), self.batch_size + with multiprocessing.Pool(self.num_proc) as pool: + for num_rows in hf_tqdm( + pool.imap( + self._batch_sql, + [(offset, index, to_sql_kwargs) for offset in range(0, num_rows, batch_size)], + ), + total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size, + unit="ba", + desc="Creating SQL from Arrow format", + ): + written += num_rows + + return written diff --git a/lib/python3.12/site-packages/datasets/io/text.py b/lib/python3.12/site-packages/datasets/io/text.py new file mode 100644 index 0000000000000000000000000000000000000000..58963f3c7ab1779d79226943e341200b782e1add --- /dev/null +++ b/lib/python3.12/site-packages/datasets/io/text.py @@ -0,0 +1,60 @@ +from typing import Optional + +from .. import Features, NamedSplit +from ..packaged_modules.text.text import Text +from ..utils.typing import NestedDataStructureLike, PathLike +from .abc import AbstractDatasetReader + + +class TextDatasetReader(AbstractDatasetReader): + def __init__( + self, + path_or_paths: NestedDataStructureLike[PathLike], + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + streaming: bool = False, + num_proc: Optional[int] = None, + **kwargs, + ): + super().__init__( + path_or_paths, + split=split, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + streaming=streaming, + num_proc=num_proc, + **kwargs, + ) + path_or_paths = path_or_paths if isinstance(path_or_paths, dict) else {self.split: path_or_paths} + self.builder = Text( + cache_dir=cache_dir, + data_files=path_or_paths, + features=features, + **kwargs, + ) + + def read(self): + # Build iterable dataset + if self.streaming: + dataset = self.builder.as_streaming_dataset(split=self.split) + # Build regular (map-style) dataset + else: + download_config = None + download_mode = None + verification_mode = None + base_path = None + + self.builder.download_and_prepare( + download_config=download_config, + download_mode=download_mode, + verification_mode=verification_mode, + base_path=base_path, + num_proc=self.num_proc, + ) + dataset = self.builder.as_dataset( + split=self.split, verification_mode=verification_mode, in_memory=self.keep_in_memory + ) + return dataset diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9faed926a2b4db0bfaf6b9697512f5dc121c8bd0 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/arrow/__init__.py b/lib/python3.12/site-packages/datasets/packaged_modules/arrow/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/arrow/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/arrow/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b6ec5a1eb1e7ba3deb6aec7eb6b801c1e4899074 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/arrow/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/arrow/__pycache__/arrow.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/arrow/__pycache__/arrow.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..401d4d9d7469b58fed85f5d1e73aa1c686c966e4 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/arrow/__pycache__/arrow.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/arrow/arrow.py b/lib/python3.12/site-packages/datasets/packaged_modules/arrow/arrow.py new file mode 100644 index 0000000000000000000000000000000000000000..bcf31c473d2ae532da908600c440a440f8a2d3d6 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/packaged_modules/arrow/arrow.py @@ -0,0 +1,79 @@ +import itertools +from dataclasses import dataclass +from typing import Optional + +import pyarrow as pa + +import datasets +from datasets.table import table_cast + + +logger = datasets.utils.logging.get_logger(__name__) + + +@dataclass +class ArrowConfig(datasets.BuilderConfig): + """BuilderConfig for Arrow.""" + + features: Optional[datasets.Features] = None + + def __post_init__(self): + super().__post_init__() + + +class Arrow(datasets.ArrowBasedBuilder): + BUILDER_CONFIG_CLASS = ArrowConfig + + def _info(self): + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager): + """We handle string, list and dicts in datafiles""" + if not self.config.data_files: + raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}") + dl_manager.download_config.extract_on_the_fly = True + data_files = dl_manager.download_and_extract(self.config.data_files) + splits = [] + for split_name, files in data_files.items(): + if isinstance(files, str): + files = [files] + # Use `dl_manager.iter_files` to skip hidden files in an extracted archive + files = [dl_manager.iter_files(file) for file in files] + # Infer features if they are stored in the arrow schema + if self.info.features is None: + for file in itertools.chain.from_iterable(files): + with open(file, "rb") as f: + try: + reader = pa.ipc.open_stream(f) + except (OSError, pa.lib.ArrowInvalid): + reader = pa.ipc.open_file(f) + self.info.features = datasets.Features.from_arrow_schema(reader.schema) + break + splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files})) + return splits + + def _cast_table(self, pa_table: pa.Table) -> pa.Table: + if self.info.features is not None: + # more expensive cast to support nested features with keys in a different order + # allows str <-> int/float or str to Audio for example + pa_table = table_cast(pa_table, self.info.features.arrow_schema) + return pa_table + + def _generate_tables(self, files): + for file_idx, file in enumerate(itertools.chain.from_iterable(files)): + with open(file, "rb") as f: + try: + try: + batches = pa.ipc.open_stream(f) + except (OSError, pa.lib.ArrowInvalid): + reader = pa.ipc.open_file(f) + batches = (reader.get_batch(i) for i in range(reader.num_record_batches)) + for batch_idx, record_batch in enumerate(batches): + pa_table = pa.Table.from_batches([record_batch]) + # Uncomment for debugging (will print the Arrow table size and elements) + # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}") + # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows))) + yield f"{file_idx}_{batch_idx}", self._cast_table(pa_table) + except ValueError as e: + logger.error(f"Failed to read file '{file}' with error {type(e)}: {e}") + raise diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/audiofolder/__init__.py b/lib/python3.12/site-packages/datasets/packaged_modules/audiofolder/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/audiofolder/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/audiofolder/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c9ec0711c238a3ed47a3262cab44b5145be09e81 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/audiofolder/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/audiofolder/__pycache__/audiofolder.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/audiofolder/__pycache__/audiofolder.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c9c9232f45ac0761a02339422cb810ab48c8ac26 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/audiofolder/__pycache__/audiofolder.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/audiofolder/audiofolder.py b/lib/python3.12/site-packages/datasets/packaged_modules/audiofolder/audiofolder.py new file mode 100644 index 0000000000000000000000000000000000000000..12ed9efdf04aea05e1b3d2d0bc01beb9f67745a8 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/packaged_modules/audiofolder/audiofolder.py @@ -0,0 +1,67 @@ +import datasets + +from ..folder_based_builder import folder_based_builder + + +logger = datasets.utils.logging.get_logger(__name__) + + +class AudioFolderConfig(folder_based_builder.FolderBasedBuilderConfig): + """Builder Config for AudioFolder.""" + + drop_labels: bool = None + drop_metadata: bool = None + + def __post_init__(self): + super().__post_init__() + + +class AudioFolder(folder_based_builder.FolderBasedBuilder): + BASE_FEATURE = datasets.Audio + BASE_COLUMN_NAME = "audio" + BUILDER_CONFIG_CLASS = AudioFolderConfig + EXTENSIONS: list[str] # definition at the bottom of the script + + +# Obtained with: +# ``` +# import soundfile as sf +# +# AUDIO_EXTENSIONS = [f".{format.lower()}" for format in sf.available_formats().keys()] +# +# # .opus decoding is supported if libsndfile >= 1.0.31: +# AUDIO_EXTENSIONS.extend([".opus"]) +# ``` +# We intentionally do not run this code on launch because: +# (1) Soundfile is an optional dependency, so importing it in global namespace is not allowed +# (2) To ensure the list of supported extensions is deterministic +AUDIO_EXTENSIONS = [ + ".aiff", + ".au", + ".avr", + ".caf", + ".flac", + ".htk", + ".svx", + ".mat4", + ".mat5", + ".mpc2k", + ".ogg", + ".paf", + ".pvf", + ".raw", + ".rf64", + ".sd2", + ".sds", + ".ircam", + ".voc", + ".w64", + ".wav", + ".nist", + ".wavex", + ".wve", + ".xi", + ".mp3", + ".opus", +] +AudioFolder.EXTENSIONS = AUDIO_EXTENSIONS diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/cache/__init__.py b/lib/python3.12/site-packages/datasets/packaged_modules/cache/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/cache/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/cache/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cc92fc29e4afb60b72e2281a3c59e5526fca019e Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/cache/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/cache/__pycache__/cache.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/cache/__pycache__/cache.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..63fe8d9cb4930eeb70a538093474b33bf45c13cf Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/cache/__pycache__/cache.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/cache/cache.py b/lib/python3.12/site-packages/datasets/packaged_modules/cache/cache.py new file mode 100644 index 0000000000000000000000000000000000000000..cdcfb4c20b6b4912e3b2e98b57cc140fe36c357a --- /dev/null +++ b/lib/python3.12/site-packages/datasets/packaged_modules/cache/cache.py @@ -0,0 +1,191 @@ +import glob +import json +import os +import shutil +import time +from pathlib import Path +from typing import Optional, Union + +import pyarrow as pa + +import datasets +import datasets.config +import datasets.data_files +from datasets.naming import camelcase_to_snakecase, filenames_for_dataset_split + + +logger = datasets.utils.logging.get_logger(__name__) + + +def _get_modification_time(cached_directory_path): + return (Path(cached_directory_path)).stat().st_mtime + + +def _find_hash_in_cache( + dataset_name: str, + config_name: Optional[str], + cache_dir: Optional[str], + config_kwargs: dict, + custom_features: Optional[datasets.Features], +) -> tuple[str, str, str]: + if config_name or config_kwargs or custom_features: + config_id = datasets.BuilderConfig(config_name or "default").create_config_id( + config_kwargs=config_kwargs, custom_features=custom_features + ) + else: + config_id = None + cache_dir = os.path.expanduser(str(cache_dir or datasets.config.HF_DATASETS_CACHE)) + namespace_and_dataset_name = dataset_name.split("/") + namespace_and_dataset_name[-1] = camelcase_to_snakecase(namespace_and_dataset_name[-1]) + cached_relative_path = "___".join(namespace_and_dataset_name) + cached_datasets_directory_path_root = os.path.join(cache_dir, cached_relative_path) + cached_directory_paths = [ + cached_directory_path + for cached_directory_path in glob.glob( + os.path.join(cached_datasets_directory_path_root, config_id or "*", "*", "*") + ) + if os.path.isdir(cached_directory_path) + and ( + config_kwargs + or custom_features + or json.loads(Path(cached_directory_path, "dataset_info.json").read_text(encoding="utf-8"))["config_name"] + == Path(cached_directory_path).parts[-3] # no extra params => config_id == config_name + ) + ] + if not cached_directory_paths: + cached_directory_paths = [ + cached_directory_path + for cached_directory_path in glob.glob(os.path.join(cached_datasets_directory_path_root, "*", "*", "*")) + if os.path.isdir(cached_directory_path) + ] + available_configs = sorted( + {Path(cached_directory_path).parts[-3] for cached_directory_path in cached_directory_paths} + ) + raise ValueError( + f"Couldn't find cache for {dataset_name}" + + (f" for config '{config_id}'" if config_id else "") + + (f"\nAvailable configs in the cache: {available_configs}" if available_configs else "") + ) + # get most recent + cached_directory_path = Path(sorted(cached_directory_paths, key=_get_modification_time)[-1]) + version, hash = cached_directory_path.parts[-2:] + other_configs = [ + Path(_cached_directory_path).parts[-3] + for _cached_directory_path in glob.glob(os.path.join(cached_datasets_directory_path_root, "*", version, hash)) + if os.path.isdir(_cached_directory_path) + and ( + config_kwargs + or custom_features + or json.loads(Path(_cached_directory_path, "dataset_info.json").read_text(encoding="utf-8"))["config_name"] + == Path(_cached_directory_path).parts[-3] # no extra params => config_id == config_name + ) + ] + if not config_id and len(other_configs) > 1: + raise ValueError( + f"There are multiple '{dataset_name}' configurations in the cache: {', '.join(other_configs)}" + f"\nPlease specify which configuration to reload from the cache, e.g." + f"\n\tload_dataset('{dataset_name}', '{other_configs[0]}')" + ) + config_name = cached_directory_path.parts[-3] + warning_msg = ( + f"Found the latest cached dataset configuration '{config_name}' at {cached_directory_path} " + f"(last modified on {time.ctime(_get_modification_time(cached_directory_path))})." + ) + logger.warning(warning_msg) + return config_name, version, hash + + +class Cache(datasets.ArrowBasedBuilder): + def __init__( + self, + cache_dir: Optional[str] = None, + dataset_name: Optional[str] = None, + config_name: Optional[str] = None, + version: Optional[str] = "0.0.0", + hash: Optional[str] = None, + base_path: Optional[str] = None, + info: Optional[datasets.DatasetInfo] = None, + features: Optional[datasets.Features] = None, + token: Optional[Union[bool, str]] = None, + repo_id: Optional[str] = None, + data_files: Optional[Union[str, list, dict, datasets.data_files.DataFilesDict]] = None, + data_dir: Optional[str] = None, + storage_options: Optional[dict] = None, + writer_batch_size: Optional[int] = None, + **config_kwargs, + ): + if repo_id is None and dataset_name is None: + raise ValueError("repo_id or dataset_name is required for the Cache dataset builder") + if data_files is not None: + config_kwargs["data_files"] = data_files + if data_dir is not None: + config_kwargs["data_dir"] = data_dir + if hash == "auto" and version == "auto": + config_name, version, hash = _find_hash_in_cache( + dataset_name=repo_id or dataset_name, + config_name=config_name, + cache_dir=cache_dir, + config_kwargs=config_kwargs, + custom_features=features, + ) + elif hash == "auto" or version == "auto": + raise NotImplementedError("Pass both hash='auto' and version='auto' instead") + super().__init__( + cache_dir=cache_dir, + dataset_name=dataset_name, + config_name=config_name, + version=version, + hash=hash, + base_path=base_path, + info=info, + token=token, + repo_id=repo_id, + storage_options=storage_options, + writer_batch_size=writer_batch_size, + ) + + def _info(self) -> datasets.DatasetInfo: + return datasets.DatasetInfo() + + def download_and_prepare(self, output_dir: Optional[str] = None, *args, **kwargs): + if not os.path.exists(self.cache_dir): + raise ValueError(f"Cache directory for {self.dataset_name} doesn't exist at {self.cache_dir}") + if output_dir is not None and output_dir != self.cache_dir: + shutil.copytree(self.cache_dir, output_dir) + + def _split_generators(self, dl_manager): + # used to stream from cache + if isinstance(self.info.splits, datasets.SplitDict): + split_infos: list[datasets.SplitInfo] = list(self.info.splits.values()) + else: + raise ValueError(f"Missing splits info for {self.dataset_name} in cache directory {self.cache_dir}") + return [ + datasets.SplitGenerator( + name=split_info.name, + gen_kwargs={ + "files": filenames_for_dataset_split( + self.cache_dir, + dataset_name=self.dataset_name, + split=split_info.name, + filetype_suffix="arrow", + shard_lengths=split_info.shard_lengths, + ) + }, + ) + for split_info in split_infos + ] + + def _generate_tables(self, files): + # used to stream from cache + for file_idx, file in enumerate(files): + with open(file, "rb") as f: + try: + for batch_idx, record_batch in enumerate(pa.ipc.open_stream(f)): + pa_table = pa.Table.from_batches([record_batch]) + # Uncomment for debugging (will print the Arrow table size and elements) + # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}") + # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows))) + yield f"{file_idx}_{batch_idx}", pa_table + except ValueError as e: + logger.error(f"Failed to read file '{file}' with error {type(e)}: {e}") + raise diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/generator/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/generator/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ea0c3f983dc04e09f1e5b53389e9b2d7e3df7917 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/generator/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/json/__init__.py b/lib/python3.12/site-packages/datasets/packaged_modules/json/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/json/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/json/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..302bab73c663a709f134d31a0a799a69eee34d33 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/json/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/json/__pycache__/json.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/json/__pycache__/json.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9c942a5e9e572f9385a928df7d1404d871b4d5ec Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/json/__pycache__/json.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/json/json.py b/lib/python3.12/site-packages/datasets/packaged_modules/json/json.py new file mode 100644 index 0000000000000000000000000000000000000000..426083fc718780eb9cafed860374ef2fe6c37dd2 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/packaged_modules/json/json.py @@ -0,0 +1,178 @@ +import io +import itertools +from dataclasses import dataclass +from typing import Optional + +import pandas as pd +import pyarrow as pa +import pyarrow.json as paj + +import datasets +import datasets.config +from datasets.table import table_cast +from datasets.utils.file_utils import readline + + +logger = datasets.utils.logging.get_logger(__name__) + + +def ujson_dumps(*args, **kwargs): + try: + return pd.io.json.ujson_dumps(*args, **kwargs) + except AttributeError: + # Before pandas-2.2.0, ujson_dumps was renamed to dumps: import ujson_dumps as dumps + return pd.io.json.dumps(*args, **kwargs) + + +def ujson_loads(*args, **kwargs): + try: + return pd.io.json.ujson_loads(*args, **kwargs) + except AttributeError: + # Before pandas-2.2.0, ujson_loads was renamed to loads: import ujson_loads as loads + return pd.io.json.loads(*args, **kwargs) + + +def pandas_read_json(path_or_buf, **kwargs): + if datasets.config.PANDAS_VERSION.major >= 2: + kwargs["dtype_backend"] = "pyarrow" + return pd.read_json(path_or_buf, **kwargs) + + +@dataclass +class JsonConfig(datasets.BuilderConfig): + """BuilderConfig for JSON.""" + + features: Optional[datasets.Features] = None + encoding: str = "utf-8" + encoding_errors: Optional[str] = None + field: Optional[str] = None + use_threads: bool = True # deprecated + block_size: Optional[int] = None # deprecated + chunksize: int = 10 << 20 # 10MB + newlines_in_values: Optional[bool] = None + + def __post_init__(self): + super().__post_init__() + + +class Json(datasets.ArrowBasedBuilder): + BUILDER_CONFIG_CLASS = JsonConfig + + def _info(self): + if self.config.block_size is not None: + logger.warning("The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead") + self.config.chunksize = self.config.block_size + if self.config.use_threads is not True: + logger.warning( + "The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore." + ) + if self.config.newlines_in_values is not None: + raise ValueError("The JSON loader parameter `newlines_in_values` is no longer supported") + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager): + """We handle string, list and dicts in datafiles""" + if not self.config.data_files: + raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}") + dl_manager.download_config.extract_on_the_fly = True + data_files = dl_manager.download_and_extract(self.config.data_files) + splits = [] + for split_name, files in data_files.items(): + if isinstance(files, str): + files = [files] + files = [dl_manager.iter_files(file) for file in files] + splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files})) + return splits + + def _cast_table(self, pa_table: pa.Table) -> pa.Table: + if self.config.features is not None: + # adding missing columns + for column_name in set(self.config.features) - set(pa_table.column_names): + type = self.config.features.arrow_schema.field(column_name).type + pa_table = pa_table.append_column(column_name, pa.array([None] * len(pa_table), type=type)) + # more expensive cast to support nested structures with keys in a different order + # allows str <-> int/float or str to Audio for example + pa_table = table_cast(pa_table, self.config.features.arrow_schema) + return pa_table + + def _generate_tables(self, files): + for file_idx, file in enumerate(itertools.chain.from_iterable(files)): + # If the file is one json object and if we need to look at the items in one specific field + if self.config.field is not None: + with open(file, encoding=self.config.encoding, errors=self.config.encoding_errors) as f: + dataset = ujson_loads(f.read()) + # We keep only the field we are interested in + dataset = dataset[self.config.field] + df = pandas_read_json(io.StringIO(ujson_dumps(dataset))) + if df.columns.tolist() == [0]: + df.columns = list(self.config.features) if self.config.features else ["text"] + pa_table = pa.Table.from_pandas(df, preserve_index=False) + yield file_idx, self._cast_table(pa_table) + + # If the file has one json object per line + else: + with open(file, "rb") as f: + batch_idx = 0 + # Use block_size equal to the chunk size divided by 32 to leverage multithreading + # Set a default minimum value of 16kB if the chunk size is really small + block_size = max(self.config.chunksize // 32, 16 << 10) + encoding_errors = ( + self.config.encoding_errors if self.config.encoding_errors is not None else "strict" + ) + while True: + batch = f.read(self.config.chunksize) + if not batch: + break + # Finish current line + try: + batch += f.readline() + except (AttributeError, io.UnsupportedOperation): + batch += readline(f) + # PyArrow only accepts utf-8 encoded bytes + if self.config.encoding != "utf-8": + batch = batch.decode(self.config.encoding, errors=encoding_errors).encode("utf-8") + try: + while True: + try: + pa_table = paj.read_json( + io.BytesIO(batch), read_options=paj.ReadOptions(block_size=block_size) + ) + break + except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e: + if ( + isinstance(e, pa.ArrowInvalid) + and "straddling" not in str(e) + or block_size > len(batch) + ): + raise + else: + # Increase the block size in case it was too small. + # The block size will be reset for the next file. + logger.debug( + f"Batch of {len(batch)} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}." + ) + block_size *= 2 + except pa.ArrowInvalid as e: + try: + with open( + file, encoding=self.config.encoding, errors=self.config.encoding_errors + ) as f: + df = pandas_read_json(f) + except ValueError: + logger.error(f"Failed to load JSON from file '{file}' with error {type(e)}: {e}") + raise e + if df.columns.tolist() == [0]: + df.columns = list(self.config.features) if self.config.features else ["text"] + try: + pa_table = pa.Table.from_pandas(df, preserve_index=False) + except pa.ArrowInvalid as e: + logger.error( + f"Failed to convert pandas DataFrame to Arrow Table from file '{file}' with error {type(e)}: {e}" + ) + raise ValueError( + f"Failed to convert pandas DataFrame to Arrow Table from file {file}." + ) from None + yield file_idx, self._cast_table(pa_table) + break + yield (file_idx, batch_idx), self._cast_table(pa_table) + batch_idx += 1 diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/pandas/__init__.py b/lib/python3.12/site-packages/datasets/packaged_modules/pandas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/pandas/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/pandas/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ff877801c637a005645ebee9244368cdda2fd34d Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/pandas/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/pandas/__pycache__/pandas.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/pandas/__pycache__/pandas.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bf11ed92cdfac88690667c2f926f58c79bf5833f Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/pandas/__pycache__/pandas.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/pandas/pandas.py b/lib/python3.12/site-packages/datasets/packaged_modules/pandas/pandas.py new file mode 100644 index 0000000000000000000000000000000000000000..d1eb50d33c899d83421167bf6dc70680e437e1e4 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/packaged_modules/pandas/pandas.py @@ -0,0 +1,65 @@ +import itertools +import warnings +from dataclasses import dataclass +from typing import Optional + +import pandas as pd +import pyarrow as pa + +import datasets +from datasets.table import table_cast + + +@dataclass +class PandasConfig(datasets.BuilderConfig): + """BuilderConfig for Pandas.""" + + features: Optional[datasets.Features] = None + + def __post_init__(self): + super().__post_init__() + + +class Pandas(datasets.ArrowBasedBuilder): + BUILDER_CONFIG_CLASS = PandasConfig + + def _info(self): + warnings.warn( + "The Pandas builder is deprecated and will be removed in the next major version of datasets.", + FutureWarning, + ) + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager): + """We handle string, list and dicts in datafiles""" + if not self.config.data_files: + raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}") + data_files = dl_manager.download_and_extract(self.config.data_files) + if isinstance(data_files, (str, list, tuple)): + files = data_files + if isinstance(files, str): + files = [files] + # Use `dl_manager.iter_files` to skip hidden files in an extracted archive + files = [dl_manager.iter_files(file) for file in files] + return [datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={"files": files})] + splits = [] + for split_name, files in data_files.items(): + if isinstance(files, str): + files = [files] + # Use `dl_manager.iter_files` to skip hidden files in an extracted archive + files = [dl_manager.iter_files(file) for file in files] + splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files})) + return splits + + def _cast_table(self, pa_table: pa.Table) -> pa.Table: + if self.config.features is not None: + # more expensive cast to support nested features with keys in a different order + # allows str <-> int/float or str to Audio for example + pa_table = table_cast(pa_table, self.config.features.arrow_schema) + return pa_table + + def _generate_tables(self, files): + for i, file in enumerate(itertools.chain.from_iterable(files)): + with open(file, "rb") as f: + pa_table = pa.Table.from_pandas(pd.read_pickle(f)) + yield i, self._cast_table(pa_table) diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/parquet/__init__.py b/lib/python3.12/site-packages/datasets/packaged_modules/parquet/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/parquet/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/parquet/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e9f8851e258b62e3e4e946c2c29e7b221b7fff74 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/parquet/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/parquet/__pycache__/parquet.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/parquet/__pycache__/parquet.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9a060eceb8453c670ab04b8f0310d4fe4bd7c90c Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/parquet/__pycache__/parquet.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/parquet/parquet.py b/lib/python3.12/site-packages/datasets/packaged_modules/parquet/parquet.py new file mode 100644 index 0000000000000000000000000000000000000000..10797753657cf04e7bf7fa6444dc5eb4d286949b --- /dev/null +++ b/lib/python3.12/site-packages/datasets/packaged_modules/parquet/parquet.py @@ -0,0 +1,109 @@ +import itertools +from dataclasses import dataclass +from typing import Optional, Union + +import pyarrow as pa +import pyarrow.dataset as ds +import pyarrow.parquet as pq + +import datasets +from datasets.table import table_cast + + +logger = datasets.utils.logging.get_logger(__name__) + + +@dataclass +class ParquetConfig(datasets.BuilderConfig): + """BuilderConfig for Parquet.""" + + batch_size: Optional[int] = None + columns: Optional[list[str]] = None + features: Optional[datasets.Features] = None + filters: Optional[Union[ds.Expression, list[tuple], list[list[tuple]]]] = None + + def __post_init__(self): + super().__post_init__() + + +class Parquet(datasets.ArrowBasedBuilder): + BUILDER_CONFIG_CLASS = ParquetConfig + + def _info(self): + if ( + self.config.columns is not None + and self.config.features is not None + and set(self.config.columns) != set(self.config.features) + ): + raise ValueError( + "The columns and features argument must contain the same columns, but got ", + f"{self.config.columns} and {self.config.features}", + ) + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager): + """We handle string, list and dicts in datafiles""" + if not self.config.data_files: + raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}") + dl_manager.download_config.extract_on_the_fly = True + data_files = dl_manager.download_and_extract(self.config.data_files) + splits = [] + for split_name, files in data_files.items(): + if isinstance(files, str): + files = [files] + # Use `dl_manager.iter_files` to skip hidden files in an extracted archive + files = [dl_manager.iter_files(file) for file in files] + # Infer features if they are stored in the arrow schema + if self.info.features is None: + for file in itertools.chain.from_iterable(files): + with open(file, "rb") as f: + self.info.features = datasets.Features.from_arrow_schema(pq.read_schema(f)) + break + splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files})) + if self.config.columns is not None and set(self.config.columns) != set(self.info.features): + self.info.features = datasets.Features( + {col: feat for col, feat in self.info.features.items() if col in self.config.columns} + ) + return splits + + def _cast_table(self, pa_table: pa.Table) -> pa.Table: + if self.info.features is not None: + # more expensive cast to support nested features with keys in a different order + # allows str <-> int/float or str to Audio for example + pa_table = table_cast(pa_table, self.info.features.arrow_schema) + return pa_table + + def _generate_tables(self, files): + if self.config.features is not None and self.config.columns is not None: + if sorted(field.name for field in self.info.features.arrow_schema) != sorted(self.config.columns): + raise ValueError( + f"Tried to load parquet data with columns '{self.config.columns}' with mismatching features '{self.info.features}'" + ) + filter_expr = ( + pq.filters_to_expression(self.config.filters) + if isinstance(self.config.filters, list) + else self.config.filters + ) + for file_idx, file in enumerate(itertools.chain.from_iterable(files)): + with open(file, "rb") as f: + parquet_fragment = ds.ParquetFileFormat().make_fragment(f) + if parquet_fragment.row_groups: + batch_size = self.config.batch_size or parquet_fragment.row_groups[0].num_rows + try: + for batch_idx, record_batch in enumerate( + parquet_fragment.to_batches( + batch_size=batch_size, + columns=self.config.columns, + filter=filter_expr, + batch_readahead=0, + fragment_readahead=0, + ) + ): + pa_table = pa.Table.from_batches([record_batch]) + # Uncomment for debugging (will print the Arrow table size and elements) + # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}") + # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows))) + yield f"{file_idx}_{batch_idx}", self._cast_table(pa_table) + except ValueError as e: + logger.error(f"Failed to read file '{file}' with error {type(e)}: {e}") + raise diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/pdffolder/__init__.py b/lib/python3.12/site-packages/datasets/packaged_modules/pdffolder/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/pdffolder/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/pdffolder/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ec24ed5df95548cf383e926c2e9a9a327ac3e59a Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/pdffolder/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/pdffolder/__pycache__/pdffolder.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/pdffolder/__pycache__/pdffolder.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bbdf3ae2d021c9db2bfcbb1bf80e8d05f8b9806e Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/pdffolder/__pycache__/pdffolder.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/pdffolder/pdffolder.py b/lib/python3.12/site-packages/datasets/packaged_modules/pdffolder/pdffolder.py new file mode 100644 index 0000000000000000000000000000000000000000..5aaf9c71a4df6551a53a2625b332bc51a2f02837 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/packaged_modules/pdffolder/pdffolder.py @@ -0,0 +1,23 @@ +import datasets + +from ..folder_based_builder import folder_based_builder + + +logger = datasets.utils.logging.get_logger(__name__) + + +class PdfFolderConfig(folder_based_builder.FolderBasedBuilderConfig): + """BuilderConfig for ImageFolder.""" + + drop_labels: bool = None + drop_metadata: bool = None + + def __post_init__(self): + super().__post_init__() + + +class PdfFolder(folder_based_builder.FolderBasedBuilder): + BASE_FEATURE = datasets.Pdf + BASE_COLUMN_NAME = "pdf" + BUILDER_CONFIG_CLASS = PdfFolderConfig + EXTENSIONS: list[str] = [".pdf"] diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/spark/__init__.py b/lib/python3.12/site-packages/datasets/packaged_modules/spark/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/spark/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/spark/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..adfd2d3f0f2e9c08d6acda964a525fe574eecf3f Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/spark/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/spark/__pycache__/spark.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/spark/__pycache__/spark.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..87aa05c608fd94b234d503033c67890a3caa67b8 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/spark/__pycache__/spark.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/spark/spark.py b/lib/python3.12/site-packages/datasets/packaged_modules/spark/spark.py new file mode 100644 index 0000000000000000000000000000000000000000..d730bfea50277dc2c4a40604cf400f4e040c20ea --- /dev/null +++ b/lib/python3.12/site-packages/datasets/packaged_modules/spark/spark.py @@ -0,0 +1,367 @@ +import os +import posixpath +import uuid +from collections.abc import Iterable +from dataclasses import dataclass +from itertools import islice +from typing import TYPE_CHECKING, Optional, Union + +import numpy as np +import pyarrow as pa + +import datasets +from datasets.arrow_writer import ArrowWriter, ParquetWriter +from datasets.config import MAX_SHARD_SIZE +from datasets.filesystems import ( + is_remote_filesystem, + rename, +) +from datasets.iterable_dataset import _BaseExamplesIterable +from datasets.utils import experimental +from datasets.utils.py_utils import convert_file_size_to_int + + +logger = datasets.utils.logging.get_logger(__name__) + +if TYPE_CHECKING: + import pyspark + import pyspark.sql + + +@dataclass +class SparkConfig(datasets.BuilderConfig): + """BuilderConfig for Spark.""" + + features: Optional[datasets.Features] = None + + def __post_init__(self): + super().__post_init__() + + +def _reorder_dataframe_by_partition(df: "pyspark.sql.DataFrame", new_partition_order: list[int]): + df_combined = df.select("*").where(f"part_id = {new_partition_order[0]}") + for partition_id in new_partition_order[1:]: + partition_df = df.select("*").where(f"part_id = {partition_id}") + df_combined = df_combined.union(partition_df) + return df_combined + + +def _generate_iterable_examples( + df: "pyspark.sql.DataFrame", + partition_order: list[int], + state_dict: Optional[dict] = None, +): + import pyspark + + df_with_partition_id = df.select("*", pyspark.sql.functions.spark_partition_id().alias("part_id")) + partition_idx_start = state_dict["partition_idx"] if state_dict else 0 + partition_df = _reorder_dataframe_by_partition(df_with_partition_id, partition_order[partition_idx_start:]) + # pipeline next partition in parallel to hide latency + rows = partition_df.toLocalIterator(prefetchPartitions=True) + curr_partition = None + row_id = state_dict["partition_example_idx"] if state_dict else 0 + for row in islice(rows, row_id, None): + row_as_dict = row.asDict() + part_id = row_as_dict["part_id"] + row_as_dict.pop("part_id") + if curr_partition != part_id: + if state_dict and curr_partition is not None: + state_dict["partition_idx"] += 1 + curr_partition = part_id + row_id = 0 + if state_dict: + state_dict["partition_example_idx"] = row_id + 1 + yield f"{part_id}_{row_id}", row_as_dict + row_id += 1 + + +class SparkExamplesIterable(_BaseExamplesIterable): + def __init__( + self, + df: "pyspark.sql.DataFrame", + partition_order=None, + ): + super().__init__() + self.df = df + self.partition_order = partition_order or range(self.df.rdd.getNumPartitions()) + + def _init_state_dict(self) -> dict: + self._state_dict = {"partition_idx": 0, "partition_example_idx": 0} + return self._state_dict + + @experimental + def load_state_dict(self, state_dict: dict) -> dict: + return super().load_state_dict(state_dict) + + def __iter__(self): + yield from _generate_iterable_examples(self.df, self.partition_order, self._state_dict) + + def shuffle_data_sources(self, generator: np.random.Generator) -> "SparkExamplesIterable": + partition_order = list(range(self.df.rdd.getNumPartitions())) + generator.shuffle(partition_order) + return SparkExamplesIterable(self.df, partition_order=partition_order) + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "SparkExamplesIterable": + partition_order = self.split_shard_indices_by_worker(num_shards=num_shards, index=index, contiguous=contiguous) + return SparkExamplesIterable(self.df, partition_order=partition_order) + + @property + def num_shards(self) -> int: + return len(self.partition_order) + + +class Spark(datasets.DatasetBuilder): + BUILDER_CONFIG_CLASS = SparkConfig + + def __init__( + self, + df: "pyspark.sql.DataFrame", + cache_dir: str = None, + working_dir: str = None, + **config_kwargs, + ): + import pyspark + + self._spark = pyspark.sql.SparkSession.builder.getOrCreate() + self.df = df + self._working_dir = working_dir + + super().__init__( + cache_dir=cache_dir, + config_name=str(self.df.semanticHash()), + **config_kwargs, + ) + + def _validate_cache_dir(self): + # Define this so that we don't reference self in create_cache_and_write_probe, which will result in a pickling + # error due to pickling the SparkContext. + cache_dir = self._cache_dir + + # Returns the path of the created file. + def create_cache_and_write_probe(context): + # makedirs with exist_ok will recursively create the directory. It will not throw an error if directories + # already exist. + os.makedirs(cache_dir, exist_ok=True) + probe_file = os.path.join(cache_dir, "fs_test" + uuid.uuid4().hex) + # Opening the file in append mode will create a new file unless it already exists, in which case it will not + # change the file contents. + open(probe_file, "a") + return [probe_file] + + if self._spark.conf.get("spark.master", "").startswith("local"): + return + + # If the cluster is multi-node, make sure that the user provided a cache_dir and that it is on an NFS + # accessible to the driver. + # TODO: Stream batches to the driver using ArrowCollectSerializer instead of throwing an error. + if self._cache_dir: + probe = ( + self._spark.sparkContext.parallelize(range(1), 1).mapPartitions(create_cache_and_write_probe).collect() + ) + if os.path.isfile(probe[0]): + return + + raise ValueError( + "When using Dataset.from_spark on a multi-node cluster, the driver and all workers should be able to access cache_dir" + ) + + def _info(self): + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager: datasets.download.download_manager.DownloadManager): + return [datasets.SplitGenerator(name=datasets.Split.TRAIN)] + + def _repartition_df_if_needed(self, max_shard_size): + import pyspark + + def get_arrow_batch_size(it): + for batch in it: + yield pa.RecordBatch.from_pydict({"batch_bytes": [batch.nbytes]}) + + df_num_rows = self.df.count() + sample_num_rows = df_num_rows if df_num_rows <= 100 else 100 + # Approximate the size of each row (in Arrow format) by averaging over a max-100-row sample. + approx_bytes_per_row = ( + self.df.limit(sample_num_rows) + .repartition(1) + .mapInArrow(get_arrow_batch_size, "batch_bytes: long") + .agg(pyspark.sql.functions.sum("batch_bytes").alias("sample_bytes")) + .collect()[0] + .sample_bytes + / sample_num_rows + ) + approx_total_size = approx_bytes_per_row * df_num_rows + if approx_total_size > max_shard_size: + # Make sure there is at least one row per partition. + new_num_partitions = min(df_num_rows, int(approx_total_size / max_shard_size)) + self.df = self.df.repartition(new_num_partitions) + + def _prepare_split_single( + self, + fpath: str, + file_format: str, + max_shard_size: int, + ) -> Iterable[tuple[int, bool, Union[int, tuple]]]: + import pyspark + + writer_class = ParquetWriter if file_format == "parquet" else ArrowWriter + working_fpath = os.path.join(self._working_dir, os.path.basename(fpath)) if self._working_dir else fpath + embed_local_files = file_format == "parquet" + + # Define these so that we don't reference self in write_arrow, which will result in a pickling error due to + # pickling the SparkContext. + features = self.config.features + writer_batch_size = self._writer_batch_size + storage_options = self._fs.storage_options + + def write_arrow(it): + # Within the same SparkContext, no two task attempts will share the same attempt ID. + task_id = pyspark.TaskContext().taskAttemptId() + first_batch = next(it, None) + if first_batch is None: + # Some partitions might not receive any data. + return pa.RecordBatch.from_arrays( + [[task_id], [0], [0]], + names=["task_id", "num_examples", "num_bytes"], + ) + shard_id = 0 + writer = writer_class( + features=features, + path=working_fpath.replace("SSSSS", f"{shard_id:05d}").replace("TTTTT", f"{task_id:05d}"), + writer_batch_size=writer_batch_size, + storage_options=storage_options, + embed_local_files=embed_local_files, + ) + table = pa.Table.from_batches([first_batch]) + writer.write_table(table) + for batch in it: + if max_shard_size is not None and writer._num_bytes >= max_shard_size: + num_examples, num_bytes = writer.finalize() + writer.close() + yield pa.RecordBatch.from_arrays( + [[task_id], [num_examples], [num_bytes]], + names=["task_id", "num_examples", "num_bytes"], + ) + shard_id += 1 + writer = writer_class( + features=writer._features, + path=working_fpath.replace("SSSSS", f"{shard_id:05d}").replace("TTTTT", f"{task_id:05d}"), + writer_batch_size=writer_batch_size, + storage_options=storage_options, + embed_local_files=embed_local_files, + ) + table = pa.Table.from_batches([batch]) + writer.write_table(table) + + if writer._num_bytes > 0: + num_examples, num_bytes = writer.finalize() + writer.close() + yield pa.RecordBatch.from_arrays( + [[task_id], [num_examples], [num_bytes]], + names=["task_id", "num_examples", "num_bytes"], + ) + + if working_fpath != fpath: + for file in os.listdir(os.path.dirname(working_fpath)): + dest = os.path.join(os.path.dirname(fpath), os.path.basename(file)) + shutil.move(file, dest) + + stats = ( + self.df.mapInArrow(write_arrow, "task_id: long, num_examples: long, num_bytes: long") + .groupBy("task_id") + .agg( + pyspark.sql.functions.sum("num_examples").alias("total_num_examples"), + pyspark.sql.functions.sum("num_bytes").alias("total_num_bytes"), + pyspark.sql.functions.count("num_bytes").alias("num_shards"), + pyspark.sql.functions.collect_list("num_examples").alias("shard_lengths"), + ) + .collect() + ) + for row in stats: + yield row.task_id, (row.total_num_examples, row.total_num_bytes, row.num_shards, row.shard_lengths) + + def _prepare_split( + self, + split_generator: "datasets.SplitGenerator", + file_format: str = "arrow", + max_shard_size: Optional[Union[str, int]] = None, + num_proc: Optional[int] = None, + **kwargs, + ): + self._validate_cache_dir() + + max_shard_size = convert_file_size_to_int(max_shard_size or MAX_SHARD_SIZE) + self._repartition_df_if_needed(max_shard_size) + is_local = not is_remote_filesystem(self._fs) + path_join = os.path.join if is_local else posixpath.join + + SUFFIX = "-TTTTT-SSSSS-of-NNNNN" + fname = f"{self.name}-{split_generator.name}{SUFFIX}.{file_format}" + fpath = path_join(self._output_dir, fname) + + total_num_examples = 0 + total_num_bytes = 0 + total_shards = 0 + task_id_and_num_shards = [] + all_shard_lengths = [] + + for task_id, content in self._prepare_split_single(fpath, file_format, max_shard_size): + ( + num_examples, + num_bytes, + num_shards, + shard_lengths, + ) = content + if num_bytes > 0: + total_num_examples += num_examples + total_num_bytes += num_bytes + total_shards += num_shards + task_id_and_num_shards.append((task_id, num_shards)) + all_shard_lengths.extend(shard_lengths) + + split_generator.split_info.num_examples = total_num_examples + split_generator.split_info.num_bytes = total_num_bytes + + # should rename everything at the end + logger.debug(f"Renaming {total_shards} shards.") + if total_shards > 1: + split_generator.split_info.shard_lengths = all_shard_lengths + + # Define fs outside of _rename_shard so that we don't reference self in the function, which will result in a + # pickling error due to pickling the SparkContext. + fs = self._fs + + # use the -SSSSS-of-NNNNN pattern + def _rename_shard( + task_id: int, + shard_id: int, + global_shard_id: int, + ): + rename( + fs, + fpath.replace("SSSSS", f"{shard_id:05d}").replace("TTTTT", f"{task_id:05d}"), + fpath.replace("TTTTT-SSSSS", f"{global_shard_id:05d}").replace("NNNNN", f"{total_shards:05d}"), + ) + + args = [] + global_shard_id = 0 + for i in range(len(task_id_and_num_shards)): + task_id, num_shards = task_id_and_num_shards[i] + for shard_id in range(num_shards): + args.append([task_id, shard_id, global_shard_id]) + global_shard_id += 1 + self._spark.sparkContext.parallelize(args, len(args)).map(lambda args: _rename_shard(*args)).collect() + else: + # don't use any pattern + shard_id = 0 + task_id = task_id_and_num_shards[0][0] + self._rename( + fpath.replace("SSSSS", f"{shard_id:05d}").replace("TTTTT", f"{task_id:05d}"), + fpath.replace(SUFFIX, ""), + ) + + def _get_examples_iterable_for_split( + self, + split_generator: "datasets.SplitGenerator", + ) -> SparkExamplesIterable: + return SparkExamplesIterable(self.df) diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/text/__init__.py b/lib/python3.12/site-packages/datasets/packaged_modules/text/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/text/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/text/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b87b4fb752e6184d10ebbc8e5140a5e8f4ae013d Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/text/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/text/__pycache__/text.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/text/__pycache__/text.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2005b20d38d0b9ac4735362ccdf5313e1d3d5b67 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/text/__pycache__/text.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/text/text.py b/lib/python3.12/site-packages/datasets/packaged_modules/text/text.py new file mode 100644 index 0000000000000000000000000000000000000000..a1f3ff5a744042abd6a9bbbb9ab074fa624a0541 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/packaged_modules/text/text.py @@ -0,0 +1,112 @@ +import itertools +from dataclasses import dataclass +from io import StringIO +from typing import Optional + +import pyarrow as pa + +import datasets +from datasets.features.features import require_storage_cast +from datasets.table import table_cast + + +logger = datasets.utils.logging.get_logger(__name__) + + +@dataclass +class TextConfig(datasets.BuilderConfig): + """BuilderConfig for text files.""" + + features: Optional[datasets.Features] = None + encoding: str = "utf-8" + encoding_errors: Optional[str] = None + chunksize: int = 10 << 20 # 10MB + keep_linebreaks: bool = False + sample_by: str = "line" + + +class Text(datasets.ArrowBasedBuilder): + BUILDER_CONFIG_CLASS = TextConfig + + def _info(self): + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager): + """The `data_files` kwarg in load_dataset() can be a str, List[str], Dict[str,str], or Dict[str,List[str]]. + + If str or List[str], then the dataset returns only the 'train' split. + If dict, then keys should be from the `datasets.Split` enum. + """ + if not self.config.data_files: + raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}") + dl_manager.download_config.extract_on_the_fly = True + data_files = dl_manager.download_and_extract(self.config.data_files) + splits = [] + for split_name, files in data_files.items(): + if isinstance(files, str): + files = [files] + files = [dl_manager.iter_files(file) for file in files] + splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files})) + return splits + + def _cast_table(self, pa_table: pa.Table) -> pa.Table: + if self.config.features is not None: + schema = self.config.features.arrow_schema + if all(not require_storage_cast(feature) for feature in self.config.features.values()): + # cheaper cast + pa_table = pa_table.cast(schema) + else: + # more expensive cast; allows str <-> int/float or str to Audio for example + pa_table = table_cast(pa_table, schema) + return pa_table + else: + return pa_table.cast(pa.schema({"text": pa.string()})) + + def _generate_tables(self, files): + pa_table_names = list(self.config.features) if self.config.features is not None else ["text"] + for file_idx, file in enumerate(itertools.chain.from_iterable(files)): + # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n" + with open(file, encoding=self.config.encoding, errors=self.config.encoding_errors) as f: + if self.config.sample_by == "line": + batch_idx = 0 + while True: + batch = f.read(self.config.chunksize) + if not batch: + break + batch += f.readline() # finish current line + # StringIO.readlines, by default splits only on "\n" (and keeps line breaks) + batch = StringIO(batch).readlines() + if not self.config.keep_linebreaks: + batch = [line.rstrip("\n") for line in batch] + pa_table = pa.Table.from_arrays([pa.array(batch)], names=pa_table_names) + # Uncomment for debugging (will print the Arrow table size and elements) + # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}") + # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows))) + yield (file_idx, batch_idx), self._cast_table(pa_table) + batch_idx += 1 + elif self.config.sample_by == "paragraph": + batch_idx = 0 + batch = "" + while True: + new_batch = f.read(self.config.chunksize) + if not new_batch: + break + batch += new_batch + batch += f.readline() # finish current line + batch = batch.split("\n\n") + pa_table = pa.Table.from_arrays( + [pa.array([example for example in batch[:-1] if example])], names=pa_table_names + ) + # Uncomment for debugging (will print the Arrow table size and elements) + # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}") + # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows))) + yield (file_idx, batch_idx), self._cast_table(pa_table) + batch_idx += 1 + batch = batch[-1] + if batch: + pa_table = pa.Table.from_arrays([pa.array([batch])], names=pa_table_names) + yield (file_idx, batch_idx), self._cast_table(pa_table) + elif self.config.sample_by == "document": + text = f.read() + pa_table = pa.Table.from_arrays([pa.array([text])], names=pa_table_names) + yield file_idx, self._cast_table(pa_table) diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/__init__.py b/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b9dd19bb612ef97131d09450cfd6cf2d3eea3c80 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/__pycache__/_tenbin.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/__pycache__/_tenbin.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bd88bcd1086acf6d88eed99134ad9d478026cef8 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/__pycache__/_tenbin.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/__pycache__/webdataset.cpython-312.pyc b/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/__pycache__/webdataset.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..83e4641938c0f335cf0774897ff7a4cac1fbf8f3 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/__pycache__/webdataset.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/_tenbin.py b/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/_tenbin.py new file mode 100644 index 0000000000000000000000000000000000000000..cd8c054842e090dc09bdf3d2fee59241a1a928c5 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/_tenbin.py @@ -0,0 +1,285 @@ +# +# Copyright (c) 2017-2021 NVIDIA CORPORATION. All rights reserved. +# This file coems from the WebDataset library. +# See the LICENSE file for licensing terms (BSD-style). +# + +""" +Binary tensor encodings for PyTorch and NumPy. + +This defines efficient binary encodings for tensors. The format is 8 byte +aligned and can be used directly for computations when transmitted, say, +via RDMA. The format is supported by WebDataset with the `.ten` filename +extension. It is also used by Tensorcom, Tensorcom RDMA, and can be used +for fast tensor storage with LMDB and in disk files (which can be memory +mapped) + +Data is encoded as a series of chunks: + +- magic number (int64) +- length in bytes (int64) +- bytes (multiple of 64 bytes long) + +Arrays are a header chunk followed by a data chunk. +Header chunks have the following structure: + +- dtype (int64) +- 8 byte array name +- ndim (int64) +- dim[0] +- dim[1] +- ... +""" + +import struct +import sys + +import numpy as np + + +def bytelen(a): + """Determine the length of a in bytes.""" + if hasattr(a, "nbytes"): + return a.nbytes + elif isinstance(a, (bytearray, bytes)): + return len(a) + else: + raise ValueError(a, "cannot determine nbytes") + + +def bytedata(a): + """Return a the raw data corresponding to a.""" + if isinstance(a, (bytearray, bytes, memoryview)): + return a + elif hasattr(a, "data"): + return a.data + else: + raise ValueError(a, "cannot return bytedata") + + +# tables for converting between long/short NumPy dtypes + +long_to_short = """ +float16 f2 +float32 f4 +float64 f8 +int8 i1 +int16 i2 +int32 i4 +int64 i8 +uint8 u1 +uint16 u2 +unit32 u4 +uint64 u8 +""".strip() +long_to_short = [x.split() for x in long_to_short.split("\n")] +long_to_short = {x[0]: x[1] for x in long_to_short} +short_to_long = {v: k for k, v in long_to_short.items()} + + +def check_acceptable_input_type(data, allow64): + """Check that the data has an acceptable type for tensor encoding. + + :param data: array + :param allow64: allow 64 bit types + """ + for a in data: + if a.dtype.name not in long_to_short: + raise ValueError("unsupported dataypte") + if not allow64 and a.dtype.name not in ["float64", "int64", "uint64"]: + raise ValueError("64 bit datatypes not allowed unless explicitly enabled") + + +def str64(s): + """Convert a string to an int64.""" + s = s + "\0" * (8 - len(s)) + s = s.encode("ascii") + return struct.unpack("@q", s)[0] + + +def unstr64(i): + """Convert an int64 to a string.""" + b = struct.pack("@q", i) + return b.decode("ascii").strip("\0") + + +def check_infos(data, infos, required_infos=None): + """Verify the info strings.""" + if required_infos is False or required_infos is None: + return data + if required_infos is True: + return data, infos + if not isinstance(required_infos, (tuple, list)): + raise ValueError("required_infos must be tuple or list") + for required, actual in zip(required_infos, infos): + raise ValueError(f"actual info {actual} doesn't match required info {required}") + return data + + +def encode_header(a, info=""): + """Encode an array header as a byte array.""" + if a.ndim >= 10: + raise ValueError("too many dimensions") + if a.nbytes != np.prod(a.shape) * a.itemsize: + raise ValueError("mismatch between size and shape") + if a.dtype.name not in long_to_short: + raise ValueError("unsupported array type") + header = [str64(long_to_short[a.dtype.name]), str64(info), len(a.shape)] + list(a.shape) + return bytedata(np.array(header, dtype="i8")) + + +def decode_header(h): + """Decode a byte array into an array header.""" + h = np.frombuffer(h, dtype="i8") + if unstr64(h[0]) not in short_to_long: + raise ValueError("unsupported array type") + dtype = np.dtype(short_to_long[unstr64(h[0])]) + info = unstr64(h[1]) + rank = int(h[2]) + shape = tuple(h[3 : 3 + rank]) + return shape, dtype, info + + +def encode_list(l, infos=None): # noqa: E741 + """Given a list of arrays, encode them into a list of byte arrays.""" + if infos is None: + infos = [""] + else: + if len(l) != len(infos): + raise ValueError(f"length of list {l} must muatch length of infos {infos}") + result = [] + for i, a in enumerate(l): + header = encode_header(a, infos[i % len(infos)]) + result += [header, bytedata(a)] + return result + + +def decode_list(l, infos=False): # noqa: E741 + """Given a list of byte arrays, decode them into arrays.""" + result = [] + infos0 = [] + for header, data in zip(l[::2], l[1::2]): + shape, dtype, info = decode_header(header) + a = np.frombuffer(data, dtype=dtype, count=np.prod(shape)).reshape(*shape) + result += [a] + infos0 += [info] + return check_infos(result, infos0, infos) + + +magic_str = "~TenBin~" +magic = str64(magic_str) +magic_bytes = unstr64(magic).encode("ascii") + + +def roundup(n, k=64): + """Round up to the next multiple of 64.""" + return k * ((n + k - 1) // k) + + +def encode_chunks(l): # noqa: E741 + """Encode a list of chunks into a single byte array, with lengths and magics..""" + size = sum(16 + roundup(b.nbytes) for b in l) + result = bytearray(size) + offset = 0 + for b in l: + result[offset : offset + 8] = magic_bytes + offset += 8 + result[offset : offset + 8] = struct.pack("@q", b.nbytes) + offset += 8 + result[offset : offset + bytelen(b)] = b + offset += roundup(bytelen(b)) + return result + + +def decode_chunks(buf): + """Decode a byte array into a list of chunks.""" + result = [] + offset = 0 + total = bytelen(buf) + while offset < total: + if magic_bytes != buf[offset : offset + 8]: + raise ValueError("magic bytes mismatch") + offset += 8 + nbytes = struct.unpack("@q", buf[offset : offset + 8])[0] + offset += 8 + b = buf[offset : offset + nbytes] + offset += roundup(nbytes) + result.append(b) + return result + + +def encode_buffer(l, infos=None): # noqa: E741 + """Encode a list of arrays into a single byte array.""" + if not isinstance(l, list): + raise ValueError("requires list") + return encode_chunks(encode_list(l, infos=infos)) + + +def decode_buffer(buf, infos=False): + """Decode a byte array into a list of arrays.""" + return decode_list(decode_chunks(buf), infos=infos) + + +def write_chunk(stream, buf): + """Write a byte chunk to the stream with magics, length, and padding.""" + nbytes = bytelen(buf) + stream.write(magic_bytes) + stream.write(struct.pack("@q", nbytes)) + stream.write(bytedata(buf)) + padding = roundup(nbytes) - nbytes + if padding > 0: + stream.write(b"\0" * padding) + + +def read_chunk(stream): + """Read a byte chunk from a stream with magics, length, and padding.""" + magic = stream.read(8) + if magic == b"": + return None + if magic != magic_bytes: + raise ValueError("magic number does not match") + nbytes = stream.read(8) + nbytes = struct.unpack("@q", nbytes)[0] + if nbytes < 0: + raise ValueError("negative nbytes") + data = stream.read(nbytes) + padding = roundup(nbytes) - nbytes + if padding > 0: + stream.read(padding) + return data + + +def write(stream, l, infos=None): # noqa: E741 + """Write a list of arrays to a stream, with magics, length, and padding.""" + for chunk in encode_list(l, infos=infos): + write_chunk(stream, chunk) + + +def read(stream, n=sys.maxsize, infos=False): + """Read a list of arrays from a stream, with magics, length, and padding.""" + chunks = [] + for _ in range(n): + header = read_chunk(stream) + if header is None: + break + data = read_chunk(stream) + if data is None: + raise ValueError("premature EOF") + chunks += [header, data] + return decode_list(chunks, infos=infos) + + +def save(fname, *args, infos=None, nocheck=False): + """Save a list of arrays to a file, with magics, length, and padding.""" + if not nocheck and not fname.endswith(".ten"): + raise ValueError("file name should end in .ten") + with open(fname, "wb") as stream: + write(stream, args, infos=infos) + + +def load(fname, infos=False, nocheck=False): + """Read a list of arrays from a file, with magics, length, and padding.""" + if not nocheck and not fname.endswith(".ten"): + raise ValueError("file name should end in .ten") + with open(fname, "rb") as stream: + return read(stream, infos=infos) diff --git a/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/webdataset.py b/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/webdataset.py new file mode 100644 index 0000000000000000000000000000000000000000..571276a4cd59377d3d2672c219558681bc9d7b94 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/webdataset.py @@ -0,0 +1,350 @@ +import io +import json +import re +from itertools import islice +from typing import Any, Callable + +import fsspec +import numpy as np +import pyarrow as pa + +import datasets +from datasets.features.features import cast_to_python_objects +from datasets.utils.file_utils import SINGLE_FILE_COMPRESSION_EXTENSION_TO_PROTOCOL, xbasename + + +logger = datasets.utils.logging.get_logger(__name__) + + +class WebDataset(datasets.GeneratorBasedBuilder): + DEFAULT_WRITER_BATCH_SIZE = 100 + IMAGE_EXTENSIONS: list[str] # definition at the bottom of the script + AUDIO_EXTENSIONS: list[str] # definition at the bottom of the script + VIDEO_EXTENSIONS: list[str] # definition at the bottom of the script + DECODERS: dict[str, Callable[[Any], Any]] # definition at the bottom of the script + NUM_EXAMPLES_FOR_FEATURES_INFERENCE = 5 + + @classmethod + def _get_pipeline_from_tar(cls, tar_path, tar_iterator): + current_example = {} + fs: fsspec.AbstractFileSystem = fsspec.filesystem("memory") + streaming_download_manager = datasets.StreamingDownloadManager() + for filename, f in tar_iterator: + example_key, field_name = base_plus_ext(filename) + if example_key is None: + continue + if current_example and current_example["__key__"] != example_key: + # reposition some keys in last position + current_example["__key__"] = current_example.pop("__key__") + current_example["__url__"] = current_example.pop("__url__") + yield current_example + current_example = {} + current_example["__key__"] = example_key + current_example["__url__"] = tar_path + current_example[field_name.lower()] = f.read() + if field_name.split(".")[-1] in SINGLE_FILE_COMPRESSION_EXTENSION_TO_PROTOCOL: + fs.write_bytes(filename, current_example[field_name.lower()]) + extracted_file_path = streaming_download_manager.extract(f"memory://{filename}") + with fsspec.open(extracted_file_path) as f: + current_example[field_name.lower()] = f.read() + fs.delete(filename) + data_extension = xbasename(extracted_file_path).split(".")[-1] + else: + data_extension = field_name.split(".")[-1] + if data_extension in cls.DECODERS: + current_example[field_name] = cls.DECODERS[data_extension](current_example[field_name]) + if current_example: + yield current_example + + def _info(self) -> datasets.DatasetInfo: + return datasets.DatasetInfo() + + def _split_generators(self, dl_manager): + """We handle string, list and dicts in datafiles""" + # Download the data files + if not self.config.data_files: + raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}") + data_files = dl_manager.download(self.config.data_files) + splits = [] + for split_name, tar_paths in data_files.items(): + if isinstance(tar_paths, str): + tar_paths = [tar_paths] + tar_iterators = [dl_manager.iter_archive(tar_path) for tar_path in tar_paths] + splits.append( + datasets.SplitGenerator( + name=split_name, gen_kwargs={"tar_paths": tar_paths, "tar_iterators": tar_iterators} + ) + ) + if not self.info.features: + # Get one example to get the feature types + pipeline = self._get_pipeline_from_tar(tar_paths[0], tar_iterators[0]) + first_examples = list(islice(pipeline, self.NUM_EXAMPLES_FOR_FEATURES_INFERENCE)) + if any(example.keys() != first_examples[0].keys() for example in first_examples): + raise ValueError( + "The TAR archives of the dataset should be in WebDataset format, " + "but the files in the archive don't share the same prefix or the same types." + ) + pa_tables = [ + pa.Table.from_pylist(cast_to_python_objects([example], only_1d_for_numpy=True)) + for example in first_examples + ] + inferred_arrow_schema = pa.concat_tables(pa_tables, promote_options="default").schema + features = datasets.Features.from_arrow_schema(inferred_arrow_schema) + + # Set Image types + for field_name in first_examples[0]: + extension = field_name.rsplit(".", 1)[-1] + if extension in self.IMAGE_EXTENSIONS: + features[field_name] = datasets.Image() + # Set Audio types + for field_name in first_examples[0]: + extension = field_name.rsplit(".", 1)[-1] + if extension in self.AUDIO_EXTENSIONS: + features[field_name] = datasets.Audio() + # Set Video types + for field_name in first_examples[0]: + extension = field_name.rsplit(".", 1)[-1] + if extension in self.VIDEO_EXTENSIONS: + features[field_name] = datasets.Video() + self.info.features = features + + return splits + + def _generate_examples(self, tar_paths, tar_iterators): + image_field_names = [ + field_name for field_name, feature in self.info.features.items() if isinstance(feature, datasets.Image) + ] + audio_field_names = [ + field_name for field_name, feature in self.info.features.items() if isinstance(feature, datasets.Audio) + ] + all_field_names = list(self.info.features.keys()) + for tar_idx, (tar_path, tar_iterator) in enumerate(zip(tar_paths, tar_iterators)): + for example_idx, example in enumerate(self._get_pipeline_from_tar(tar_path, tar_iterator)): + for field_name in all_field_names: + if field_name not in example: + example[field_name] = None + for field_name in image_field_names + audio_field_names: + if example[field_name] is not None: + example[field_name] = { + "path": example["__key__"] + "." + field_name, + "bytes": example[field_name], + } + yield f"{tar_idx}_{example_idx}", example + + +# Source: https://github.com/webdataset/webdataset/blob/87bd5aa41602d57f070f65a670893ee625702f2f/webdataset/tariterators.py#L25 +def base_plus_ext(path): + """Split off all file extensions. + + Returns base, allext. + """ + match = re.match(r"^((?:.*/|)[^.]+)[.]([^/]*)$", path) + if not match: + return None, None + return match.group(1), match.group(2) + + +# Obtained with: +# ``` +# import PIL.Image +# IMAGE_EXTENSIONS = [] +# PIL.Image.init() +# for ext, format in PIL.Image.EXTENSION.items(): +# if format in PIL.Image.OPEN: +# IMAGE_EXTENSIONS.append(ext[1:]) +# ``` +# We intentionally do not run this code on launch because: +# (1) Pillow is an optional dependency, so importing Pillow in global namespace is not allowed +# (2) To ensure the list of supported extensions is deterministic +IMAGE_EXTENSIONS = [ + "blp", + "bmp", + "dib", + "bufr", + "cur", + "pcx", + "dcx", + "dds", + "ps", + "eps", + "fit", + "fits", + "fli", + "flc", + "ftc", + "ftu", + "gbr", + "gif", + "grib", + "h5", + "hdf", + "png", + "apng", + "jp2", + "j2k", + "jpc", + "jpf", + "jpx", + "j2c", + "icns", + "ico", + "im", + "iim", + "tif", + "tiff", + "jfif", + "jpe", + "jpg", + "jpeg", + "mpg", + "mpeg", + "msp", + "pcd", + "pxr", + "pbm", + "pgm", + "ppm", + "pnm", + "psd", + "bw", + "rgb", + "rgba", + "sgi", + "ras", + "tga", + "icb", + "vda", + "vst", + "webp", + "wmf", + "emf", + "xbm", + "xpm", +] +WebDataset.IMAGE_EXTENSIONS = IMAGE_EXTENSIONS + + +# Obtained with: +# ``` +# import soundfile as sf +# +# AUDIO_EXTENSIONS = [f".{format.lower()}" for format in sf.available_formats().keys()] +# +# # .opus decoding is supported if libsndfile >= 1.0.31: +# AUDIO_EXTENSIONS.extend([".mp3", ".opus"]) +# ``` +# We intentionally do not run this code on launch because: +# (1) Soundfile is an optional dependency, so importing it in global namespace is not allowed +# (2) To ensure the list of supported extensions is deterministic +AUDIO_EXTENSIONS = [ + "aiff", + "au", + "avr", + "caf", + "flac", + "htk", + "svx", + "mat4", + "mat5", + "mpc2k", + "ogg", + "paf", + "pvf", + "raw", + "rf64", + "sd2", + "sds", + "ircam", + "voc", + "w64", + "wav", + "nist", + "wavex", + "wve", + "xi", + "mp3", + "opus", +] +WebDataset.AUDIO_EXTENSIONS = AUDIO_EXTENSIONS + + +# TODO: initial list, we should check the compatibility of other formats +VIDEO_EXTENSIONS = [ + ".mkv", + ".mp4", + ".avi", + ".mpeg", + ".mov", +] +WebDataset.VIDEO_EXTENSIONS = VIDEO_EXTENSIONS + + +def text_loads(data: bytes): + return data.decode("utf-8") + + +def tenbin_loads(data: bytes): + from . import _tenbin + + return _tenbin.decode_buffer(data) + + +def msgpack_loads(data: bytes): + import msgpack + + return msgpack.unpackb(data) + + +def npy_loads(data: bytes): + import numpy.lib.format + + stream = io.BytesIO(data) + return numpy.lib.format.read_array(stream, allow_pickle=False) + + +def npz_loads(data: bytes): + return np.load(io.BytesIO(data), allow_pickle=False) + + +def cbor_loads(data: bytes): + import cbor + + return cbor.loads(data) + + +def torch_loads(data: bytes): + import torch + + return torch.load(io.BytesIO(data), weights_only=True) + + +# Obtained by checking `decoders` in `webdataset.autodecode` +# and removing unsafe extension decoders. +# Removed Pickle decoders: +# - "pyd": lambda data: pickle.loads(data) +# - "pickle": lambda data: pickle.loads(data) +# Modified NumPy decoders to fix CVE-2019-6446 (add allow_pickle=False and weights_only=True): +# - "npy": npy_loads, +# - "npz": lambda data: np.load(io.BytesIO(data)), +# - "pth": lambda data: torch_loads(data) +DECODERS = { + "txt": text_loads, + "text": text_loads, + "transcript": text_loads, + "cls": int, + "cls2": int, + "index": int, + "inx": int, + "id": int, + "json": json.loads, + "jsn": json.loads, + "ten": tenbin_loads, + "tb": tenbin_loads, + "mp": msgpack_loads, + "msg": msgpack_loads, + "npy": npy_loads, + "npz": npz_loads, + "cbor": cbor_loads, + "pth": torch_loads, +} +WebDataset.DECODERS = DECODERS diff --git a/lib/python3.12/site-packages/datasets/parallel/__init__.py b/lib/python3.12/site-packages/datasets/parallel/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a8d527f4d09946b3b7b7a65ec22afa953ddfd315 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/parallel/__init__.py @@ -0,0 +1 @@ +from .parallel import ParallelBackendConfig, parallel_backend, parallel_map diff --git a/lib/python3.12/site-packages/datasets/parallel/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/datasets/parallel/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9cba7c6b5e4625a20005dbaecca86843cc54f76b Binary files /dev/null and b/lib/python3.12/site-packages/datasets/parallel/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/parallel/__pycache__/parallel.cpython-312.pyc b/lib/python3.12/site-packages/datasets/parallel/__pycache__/parallel.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..95d84dc5f2759a28e8d80c378b53594947315f91 Binary files /dev/null and b/lib/python3.12/site-packages/datasets/parallel/__pycache__/parallel.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/datasets/parallel/parallel.py b/lib/python3.12/site-packages/datasets/parallel/parallel.py new file mode 100644 index 0000000000000000000000000000000000000000..5cad2c48ba2b0a6e71c62babd4eb052a3dd1dfd4 --- /dev/null +++ b/lib/python3.12/site-packages/datasets/parallel/parallel.py @@ -0,0 +1,120 @@ +import contextlib +from multiprocessing import Pool, RLock + +from tqdm.auto import tqdm + +from ..utils import experimental, logging + + +logger = logging.get_logger(__name__) + + +class ParallelBackendConfig: + backend_name = None + + +@experimental +def parallel_map(function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, single_map_nested_func): + """ + **Experimental.** Apply a function to iterable elements in parallel, where the implementation uses either + multiprocessing.Pool or joblib for parallelization. + + Args: + function (`Callable[[Any], Any]`): Function to be applied to `iterable`. + iterable (`list`, `tuple` or `np.ndarray`): Iterable elements to apply function to. + num_proc (`int`): Number of processes (if no backend specified) or jobs (using joblib). + types (`tuple`): Additional types (besides `dict` values) to apply `function` recursively to their elements. + disable_tqdm (`bool`): Whether to disable the tqdm progressbar. + desc (`str`): Prefix for the tqdm progressbar. + single_map_nested_func (`Callable`): Map function that applies `function` to an element from `iterable`. + Takes a tuple of function, data_struct, types, rank, disable_tqdm, desc as input, where data_struct is an + element of `iterable`, and `rank` is used for progress bar. + """ + if ParallelBackendConfig.backend_name is None: + return _map_with_multiprocessing_pool( + function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, single_map_nested_func + ) + + return _map_with_joblib( + function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, single_map_nested_func + ) + + +def _map_with_multiprocessing_pool( + function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, single_map_nested_func +): + num_proc = num_proc if num_proc <= len(iterable) else len(iterable) + split_kwds = [] # We organize the splits ourselve (contiguous splits) + for index in range(num_proc): + div = len(iterable) // num_proc + mod = len(iterable) % num_proc + start = div * index + min(index, mod) + end = start + div + (1 if index < mod else 0) + split_kwds.append((function, iterable[start:end], batched, batch_size, types, index, disable_tqdm, desc)) + + if len(iterable) != sum(len(i[1]) for i in split_kwds): + raise ValueError( + f"Error dividing inputs iterable among processes. " + f"Total number of objects {len(iterable)}, " + f"length: {sum(len(i[1]) for i in split_kwds)}" + ) + + logger.info( + f"Spawning {num_proc} processes for {len(iterable)} objects in slices of {[len(i[1]) for i in split_kwds]}" + ) + initargs, initializer = None, None + if not disable_tqdm: + initargs, initializer = (RLock(),), tqdm.set_lock + with Pool(num_proc, initargs=initargs, initializer=initializer) as pool: + mapped = pool.map(single_map_nested_func, split_kwds) + logger.info(f"Finished {num_proc} processes") + mapped = [obj for proc_res in mapped for obj in proc_res] + logger.info(f"Unpacked {len(mapped)} objects") + + return mapped + + +def _map_with_joblib( + function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, single_map_nested_func +): + # progress bar is not yet supported for _map_with_joblib, because tqdm couldn't accurately be applied to joblib, + # and it requires monkey-patching joblib internal classes which is subject to change + import joblib + + with joblib.parallel_backend(ParallelBackendConfig.backend_name, n_jobs=num_proc): + return joblib.Parallel()( + joblib.delayed(single_map_nested_func)((function, obj, batched, batch_size, types, None, True, None)) + for obj in iterable + ) + + +@experimental +@contextlib.contextmanager +def parallel_backend(backend_name: str): + """ + **Experimental.** Configures the parallel backend for parallelized dataset loading, which uses the parallelization + implemented by joblib. + + Args: + backend_name (str): Name of backend for parallelization implementation, has to be supported by joblib. + + Example usage: + ```py + with parallel_backend('spark'): + dataset = load_dataset(..., num_proc=2) + ``` + """ + ParallelBackendConfig.backend_name = backend_name + + if backend_name == "spark": + from joblibspark import register_spark + + register_spark() + + # TODO: call create_cache_and_write_probe if "download" in steps + # TODO: raise NotImplementedError when Dataset.map etc is called + + try: + yield + finally: + ParallelBackendConfig.backend_name = None diff --git a/lib/python3.12/site-packages/executing/__init__.py b/lib/python3.12/site-packages/executing/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e5181a5c326ce2267ebc5847ed809d0c0ede5865 --- /dev/null +++ b/lib/python3.12/site-packages/executing/__init__.py @@ -0,0 +1,28 @@ +""" +Get information about what a frame is currently doing. Typical usage: + + import executing + + node = executing.Source.executing(frame).node + # node will be an AST node or None +""" + +from collections import namedtuple +_VersionInfo = namedtuple('_VersionInfo', ('major', 'minor', 'micro')) +from .executing import Source, Executing, only, NotOneValueFound, cache, future_flags + +from ._pytest_utils import is_pytest_compatible + +try: + from .version import __version__ # type: ignore[import] + if "dev" in __version__: + raise ValueError +except Exception: + # version.py is auto-generated with the git tag when building + __version__ = "???" + __version_info__ = _VersionInfo(-1, -1, -1) +else: + __version_info__ = _VersionInfo(*map(int, __version__.split('.'))) + + +__all__ = ["Source","is_pytest_compatible"] diff --git a/lib/python3.12/site-packages/executing/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/executing/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cb5c74aea0cce5c0cd0a419cec8dd1e87caf97bb Binary files /dev/null and b/lib/python3.12/site-packages/executing/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/executing/__pycache__/_exceptions.cpython-312.pyc b/lib/python3.12/site-packages/executing/__pycache__/_exceptions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6f17931ba385b3eade7daaafa5c572a53a0696ec Binary files /dev/null and b/lib/python3.12/site-packages/executing/__pycache__/_exceptions.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/executing/__pycache__/_position_node_finder.cpython-312.pyc b/lib/python3.12/site-packages/executing/__pycache__/_position_node_finder.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d0d29bde91ec7e9963d8ded13752053dfbcff34d Binary files /dev/null and b/lib/python3.12/site-packages/executing/__pycache__/_position_node_finder.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/executing/__pycache__/_pytest_utils.cpython-312.pyc b/lib/python3.12/site-packages/executing/__pycache__/_pytest_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b486a9ed512fcfc9fe393e711f9fc758a357df68 Binary files /dev/null and b/lib/python3.12/site-packages/executing/__pycache__/_pytest_utils.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/executing/__pycache__/_utils.cpython-312.pyc b/lib/python3.12/site-packages/executing/__pycache__/_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5c28ff66eb5009ae70109070dff154d3ecfe9c6f Binary files /dev/null and b/lib/python3.12/site-packages/executing/__pycache__/_utils.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/executing/__pycache__/executing.cpython-312.pyc b/lib/python3.12/site-packages/executing/__pycache__/executing.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7a252d0ea60c286fb806f85a2cfa0c4105be9a01 Binary files /dev/null and b/lib/python3.12/site-packages/executing/__pycache__/executing.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/executing/__pycache__/version.cpython-312.pyc b/lib/python3.12/site-packages/executing/__pycache__/version.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..57d56304bb6568d561f230c2f9fbb37be46ecf3e Binary files /dev/null and b/lib/python3.12/site-packages/executing/__pycache__/version.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/executing/_exceptions.py b/lib/python3.12/site-packages/executing/_exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..1e88e3f29c9e16bc9776e47f5ff746e680ca5351 --- /dev/null +++ b/lib/python3.12/site-packages/executing/_exceptions.py @@ -0,0 +1,22 @@ + +class KnownIssue(Exception): + """ + Raised in case of an known problem. Mostly because of cpython bugs. + Executing.node gets set to None in this case. + """ + + pass + + +class VerifierFailure(Exception): + """ + Thrown for an unexpected mapping from instruction to ast node + Executing.node gets set to None in this case. + """ + + def __init__(self, title, node, instruction): + # type: (object, object, object) -> None + self.node = node + self.instruction = instruction + + super().__init__(title) # type: ignore[call-arg] diff --git a/lib/python3.12/site-packages/executing/_position_node_finder.py b/lib/python3.12/site-packages/executing/_position_node_finder.py new file mode 100644 index 0000000000000000000000000000000000000000..27028d26ec99b9b3e60757fda567497d50ec6933 --- /dev/null +++ b/lib/python3.12/site-packages/executing/_position_node_finder.py @@ -0,0 +1,1045 @@ +import ast +import sys +import dis +from types import CodeType, FrameType +from typing import Any, Callable, Iterator, Optional, Sequence, Set, Tuple, Type, Union, cast +from .executing import EnhancedAST, NotOneValueFound, Source, only, function_node_types, assert_ +from ._exceptions import KnownIssue, VerifierFailure +from ._utils import mangled_name + +from functools import lru_cache +import itertools + +# the code in this module can use all python>=3.11 features + + +def parents(node: EnhancedAST) -> Iterator[EnhancedAST]: + while True: + if hasattr(node, "parent"): + node = node.parent + yield node + else: + break # pragma: no mutate + + +def node_and_parents(node: EnhancedAST) -> Iterator[EnhancedAST]: + yield node + yield from parents(node) + + +@lru_cache(128) # pragma: no mutate +def get_instructions(code: CodeType) -> list[dis.Instruction]: + return list(dis.get_instructions(code)) + + +types_cmp_issue_fix = ( + ast.IfExp, + ast.If, + ast.Assert, + ast.While, +) + +types_cmp_issue = types_cmp_issue_fix + ( + ast.ListComp, + ast.SetComp, + ast.DictComp, + ast.GeneratorExp, +) + +op_type_map = { + "**": ast.Pow, + "*": ast.Mult, + "@": ast.MatMult, + "//": ast.FloorDiv, + "/": ast.Div, + "%": ast.Mod, + "+": ast.Add, + "-": ast.Sub, + "<<": ast.LShift, + ">>": ast.RShift, + "&": ast.BitAnd, + "^": ast.BitXor, + "|": ast.BitOr, +} + + +class PositionNodeFinder(object): + """ + Mapping bytecode to ast-node based on the source positions, which where introduced in pyhon 3.11. + In general every ast-node can be exactly referenced by its begin/end line/col_offset, which is stored in the bytecode. + There are only some exceptions for methods and attributes. + """ + + def __init__(self, frame: FrameType, stmts: Set[EnhancedAST], tree: ast.Module, lasti: int, source: Source): + self.bc_dict={bc.offset:bc for bc in get_instructions(frame.f_code) } + self.frame=frame + + self.source = source + self.decorator: Optional[EnhancedAST] = None + + # work around for https://github.com/python/cpython/issues/96970 + while self.opname(lasti) == "CACHE": + lasti -= 2 + + try: + # try to map with all match_positions + self.result = self.find_node(lasti) + except NotOneValueFound: + typ: tuple[Type] + # LOAD_METHOD could load "".join for long "..."%(...) BinOps + # this can only be associated by using all positions + if self.opname(lasti) in ( + "LOAD_METHOD", + "LOAD_ATTR", + "STORE_ATTR", + "DELETE_ATTR", + ): + # lineno and col_offset of LOAD_METHOD and *_ATTR instructions get set to the beginning of + # the attribute by the python compiler to improved error messages (PEP-657) + # we ignore here the start position and try to find the ast-node just by end position and expected node type + # This is save, because there can only be one attribute ending at a specific point in the source code. + typ = (ast.Attribute,) + elif self.opname(lasti) in ("CALL", "CALL_KW"): + # A CALL instruction can be a method call, in which case the lineno and col_offset gets changed by the compiler. + # Therefore we ignoring here this attributes and searchnig for a Call-node only by end_col_offset and end_lineno. + # This is save, because there can only be one method ending at a specific point in the source code. + # One closing ) only belongs to one method. + typ = (ast.Call,) + else: + raise + + self.result = self.find_node( + lasti, + match_positions=("end_col_offset", "end_lineno"), + typ=typ, + ) + + instruction = self.instruction(lasti) + assert instruction is not None + + self.result = self.fix_result(self.result, instruction) + + self.known_issues(self.result, instruction) + + self.test_for_decorator(self.result, lasti) + + # verify + if self.decorator is None: + self.verify(self.result, instruction) + else: + assert_(self.decorator in self.result.decorator_list) + + def test_for_decorator(self, node: EnhancedAST, index: int) -> None: + if ( + isinstance(node.parent, (ast.ClassDef, function_node_types)) + and node in node.parent.decorator_list # type: ignore[attr-defined] + ): + node_func = node.parent + + while True: + # the generated bytecode looks like follow: + + # index opname + # ------------------ + # index-4 PRECALL (only in 3.11) + # index-2 CACHE + # index CALL <- the call instruction + # ... CACHE some CACHE instructions + + # maybe multiple other bytecode blocks for other decorators + # index-4 PRECALL (only in 3.11) + # index-2 CACHE + # index CALL <- index of the next loop + # ... CACHE some CACHE instructions + + # index+x STORE_* the ast-node of this instruction points to the decorated thing + + if not ( + (self.opname(index - 4) == "PRECALL" or sys.version_info >= (3, 12)) + and self.opname(index) == "CALL" + ): # pragma: no mutate + break # pragma: no mutate + + index += 2 + + while self.opname(index) in ("CACHE", "EXTENDED_ARG"): + index += 2 + + if ( + self.opname(index).startswith("STORE_") + and self.find_node(index) == node_func + ): + self.result = node_func + self.decorator = node + return + + if sys.version_info < (3, 12): + index += 4 + + def fix_result( + self, node: EnhancedAST, instruction: dis.Instruction + ) -> EnhancedAST: + if ( + sys.version_info >= (3, 12, 5) + and instruction.opname in ("GET_ITER", "FOR_ITER") + and isinstance(node.parent, ast.For) + and node is node.parent.iter + ): + # node positions have changed in 3.12.5 + # https://github.com/python/cpython/issues/93691 + # `for` calls __iter__ and __next__ during execution, the calling + # expression of these calls was the ast.For node since cpython 3.11 (see test_iter). + # cpython 3.12.5 changed this to the `iter` node of the loop, to make tracebacks easier to read. + # This keeps backward compatibility with older executing versions. + + # there are also cases like: + # + # for a in iter(l): pass + # + # where `iter(l)` would be otherwise the resulting node for the `iter()` call and the __iter__ call of the for implementation. + # keeping the old behaviour makes it possible to distinguish both cases. + + return node.parent + + if ( + sys.version_info >= (3, 12, 6) + and instruction.opname in ("GET_ITER", "FOR_ITER") + and isinstance( + node.parent.parent, + (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp), + ) + and isinstance(node.parent,ast.comprehension) + and node is node.parent.iter + ): + # same as above but only for comprehensions, see: + # https://github.com/python/cpython/issues/123142 + + return node.parent.parent + + if sys.version_info >= (3, 12,6) and instruction.opname == "CALL": + before = self.instruction_before(instruction) + if ( + before is not None + and before.opname == "LOAD_CONST" + and before.positions == instruction.positions + and isinstance(node.parent, ast.withitem) + and node is node.parent.context_expr + ): + # node positions for with-statements have change + # and is now equal to the expression which created the context-manager + # https://github.com/python/cpython/pull/120763 + + # with context_manager: + # ... + + # but there is one problem to distinguish call-expressions from __exit__() + + # with context_manager(): + # ... + + # the call for __exit__ + + # 20 1:5 1:22 LOAD_CONST(None) + # 22 1:5 1:22 LOAD_CONST(None) + # 24 1:5 1:22 LOAD_CONST(None) + # 26 1:5 1:22 CALL() # <-- same source range as context_manager() + + # but we can use the fact that the previous load for None + # has the same source range as the call, wich can not happen for normal calls + + # we return the same ast.With statement at the and to preserve backward compatibility + + return node.parent.parent + + if ( + sys.version_info >= (3, 12,6) + and instruction.opname == "BEFORE_WITH" + and isinstance(node.parent, ast.withitem) + and node is node.parent.context_expr + ): + # handle positions changes for __enter__ + return node.parent.parent + + if sys.version_info >= (3, 14) and instruction.opname == "CALL": + before = self.instruction_before(instruction) + if ( + before is not None + and before.opname == "LOAD_SPECIAL" + and before.argrepr in ("__enter__","__aenter__") + and before.positions == instruction.positions + and isinstance(node.parent, ast.withitem) + and node is node.parent.context_expr + ): + return node.parent.parent + + if sys.version_info >= (3, 14) and isinstance(node, ast.UnaryOp) and isinstance(node.op,ast.Not) and instruction.opname !="UNARY_NOT": + # fix for https://github.com/python/cpython/issues/137843 + return node.operand + + + return node + + def known_issues(self, node: EnhancedAST, instruction: dis.Instruction) -> None: + if instruction.opname in ("COMPARE_OP", "IS_OP", "CONTAINS_OP") and isinstance( + node, types_cmp_issue + ): + if isinstance(node, types_cmp_issue_fix): + # this is a workaround for https://github.com/python/cpython/issues/95921 + # we can fix cases with only on comparison inside the test condition + # + # we can not fix cases like: + # if a 1 + ] + + assert_(comparisons, "expected at least one comparison") + + if len(comparisons) == 1: + node = self.result = cast(EnhancedAST, comparisons[0]) + else: + raise KnownIssue( + "multiple chain comparison inside %s can not be fixed" % (node) + ) + + else: + # Comprehension and generators get not fixed for now. + raise KnownIssue("chain comparison inside %s can not be fixed" % (node)) + + if ( + sys.version_info[:3] == (3, 11, 1) + and isinstance(node, ast.Compare) + and instruction.opname == "CALL" + and any(isinstance(n, ast.Assert) for n in node_and_parents(node)) + ): + raise KnownIssue( + "known bug in 3.11.1 https://github.com/python/cpython/issues/95921" + ) + + if isinstance(node, ast.Assert): + # pytest assigns the position of the assertion to all expressions of the rewritten assertion. + # All the rewritten expressions get mapped to ast.Assert, which is the wrong ast-node. + # We don't report this wrong result. + raise KnownIssue("assert") + + if any(isinstance(n, ast.pattern) for n in node_and_parents(node)): + # TODO: investigate + raise KnownIssue("pattern matching ranges seems to be wrong") + + if ( + sys.version_info >= (3, 12) + and isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "super" + ): + # super is optimized to some instructions which do not map nicely to a Call + + # find the enclosing function + func = node.parent + while hasattr(func, "parent") and not isinstance( + func, (ast.AsyncFunctionDef, ast.FunctionDef) + ): + + func = func.parent + + # get the first function argument (self/cls) + first_arg = None + + if hasattr(func, "args"): + args = [*func.args.posonlyargs, *func.args.args] + if args: + first_arg = args[0].arg + + if (instruction.opname, instruction.argval) in [ + ("LOAD_DEREF", "__class__"), + ("LOAD_FAST", first_arg), + ("LOAD_FAST_BORROW", first_arg), + ("LOAD_DEREF", first_arg), + ]: + raise KnownIssue("super optimization") + + if self.is_except_cleanup(instruction, node): + raise KnownIssue("exeption cleanup does not belong to the last node in a except block") + + if instruction.opname == "STORE_NAME" and instruction.argval == "__classcell__": + # handle stores to __classcell__ as KnownIssue, + # because they get complicated if they are used in `if` or `for` loops + # example: + # + # class X: + # # ... something + # if some_condition: + # def method(self): + # super() + # + # The `STORE_NAME` instruction gets mapped to the `ast.If` node, + # because it is the last element in the class. + # This last element could be anything and gets dificult to verify. + + raise KnownIssue("store __classcell__") + + if ( + instruction.opname == "CALL" + and not isinstance(node,ast.Call) + and any(isinstance(p, ast.Assert) for p in parents(node)) + and sys.version_info >= (3, 11, 2) + ): + raise KnownIssue("exception generation maps to condition") + + if sys.version_info >= (3, 13): + if instruction.opname in ( + "STORE_FAST_STORE_FAST", + "STORE_FAST_LOAD_FAST", + "LOAD_FAST_LOAD_FAST", + ): + raise KnownIssue(f"can not map {instruction.opname} to two ast nodes") + + if instruction.opname in ("LOAD_FAST","LOAD_FAST_BORROW") and instruction.argval == "__class__": + # example: + # class T: + # def a(): + # super() + # some_node # <- there is a LOAD_FAST for this node because we use super() + + raise KnownIssue( + f"loading of __class__ is accociated with a random node at the end of a class if you use super()" + ) + + if ( + instruction.opname == "COMPARE_OP" + and isinstance(node, ast.UnaryOp) + and isinstance(node.operand,ast.Compare) + and isinstance(node.op, ast.Not) + ): + # work around for + # https://github.com/python/cpython/issues/114671 + self.result = node.operand + + if sys.version_info >= (3,14): + + + if header_length := self.annotation_header_size(): + + last_offset=list(self.bc_dict.keys())[-1] + if ( + not (header_length*2 < instruction.offset int: + if sys.version_info >=(3,14): + header=[inst.opname for inst in itertools.islice(self.bc_dict.values(),8)] + + if len(header)==8: + if header[0] in ("COPY_FREE_VARS","MAKE_CELL"): + del header[0] + header_size=8 + else: + del header[7] + header_size=7 + + if header==[ + "RESUME", + "LOAD_FAST_BORROW", + "LOAD_SMALL_INT", + "COMPARE_OP", + "POP_JUMP_IF_FALSE", + "NOT_TAKEN", + "LOAD_COMMON_CONSTANT", + ]: + return header_size + + return 0 + + @staticmethod + def is_except_cleanup(inst: dis.Instruction, node: EnhancedAST) -> bool: + if inst.opname not in ( + "STORE_NAME", + "STORE_FAST", + "STORE_DEREF", + "STORE_GLOBAL", + "DELETE_NAME", + "DELETE_FAST", + "DELETE_DEREF", + "DELETE_GLOBAL", + ): + return False + + # This bytecode does something exception cleanup related. + # The position of the instruciton seems to be something in the last ast-node of the ExceptHandler + # this could be a bug, but it might not be observable in normal python code. + + # example: + # except Exception as exc: + # enum_member._value_ = value + + # other example: + # STORE_FAST of e was mapped to Constant(value=False) + # except OSError as e: + # if not _ignore_error(e): + # raise + # return False + + # STORE_FAST of msg was mapped to print(...) + # except TypeError as msg: + # print("Sorry:", msg, file=file) + + if ( + isinstance(node, ast.Name) + and isinstance(node.ctx,ast.Store) + and inst.opname.startswith("STORE_") + and mangled_name(node) == inst.argval + ): + # Storing the variable is valid and no exception cleanup, if the name is correct + return False + + if ( + isinstance(node, ast.Name) + and isinstance(node.ctx,ast.Del) + and inst.opname.startswith("DELETE_") + and mangled_name(node) == inst.argval + ): + # Deleting the variable is valid and no exception cleanup, if the name is correct + return False + + return any( + isinstance(n, ast.ExceptHandler) and n.name and mangled_name(n) == inst.argval + for n in parents(node) + ) + + def verify(self, node: EnhancedAST, instruction: dis.Instruction) -> None: + """ + checks if this node could gererate this instruction + """ + + op_name = instruction.opname + extra_filter: Callable[[EnhancedAST], bool] = lambda e: True + ctx: Type = type(None) + + def inst_match(opnames: Union[str, Sequence[str]], **kwargs: Any) -> bool: + """ + match instruction + + Parameters: + opnames: (str|Seq[str]): inst.opname has to be equal to or in `opname` + **kwargs: every arg has to match inst.arg + + Returns: + True if all conditions match the instruction + + """ + + if isinstance(opnames, str): + opnames = [opnames] + return instruction.opname in opnames and kwargs == { + k: getattr(instruction, k) for k in kwargs + } + + def node_match(node_type: Union[Type, Tuple[Type, ...]], **kwargs: Any) -> bool: + """ + match the ast-node + + Parameters: + node_type: type of the node + **kwargs: every `arg` has to be equal `node.arg` + or `node.arg` has to be an instance of `arg` if it is a type. + """ + return isinstance(node, node_type) and all( + isinstance(getattr(node, k), v) + if isinstance(v, type) + else getattr(node, k) == v + for k, v in kwargs.items() + ) + + if op_name == "CACHE": + return + + if inst_match("CALL") and node_match((ast.With, ast.AsyncWith)): + # call to context.__exit__ + return + + if inst_match(("CALL", "LOAD_FAST","LOAD_FAST_BORROW")) and node_match( + (ast.ListComp, ast.GeneratorExp, ast.SetComp, ast.DictComp) + ): + # call to the generator function + return + + if ( + sys.version_info >= (3, 12) + and inst_match(("LOAD_FAST_AND_CLEAR", "STORE_FAST")) + and node_match((ast.ListComp, ast.SetComp, ast.DictComp)) + ): + return + + if inst_match(("CALL", "CALL_FUNCTION_EX")) and node_match( + (ast.ClassDef, ast.Call) + ): + return + + if inst_match(("COMPARE_OP", "IS_OP", "CONTAINS_OP")) and node_match( + ast.Compare + ): + return + + if inst_match("LOAD_NAME", argval="__annotations__") and node_match( + ast.AnnAssign + ): + return + + if ( + ( + inst_match("LOAD_METHOD", argval="join") + or inst_match("LOAD_ATTR", argval="join") # 3.12 + or inst_match(("CALL", "BUILD_STRING")) + ) + and node_match(ast.BinOp, left=ast.Constant, op=ast.Mod) + and isinstance(cast(ast.Constant, cast(ast.BinOp, node).left).value, str) + ): + # "..."%(...) uses "".join + return + + if inst_match("STORE_SUBSCR") and node_match(ast.AnnAssign): + # data: int + return + + + if inst_match(("DELETE_NAME", "DELETE_FAST")) and node_match( + ast.Name, id=instruction.argval, ctx=ast.Del + ): + return + + if inst_match("BUILD_STRING") and ( + node_match(ast.JoinedStr) or node_match(ast.BinOp, op=ast.Mod) + ): + return + + if inst_match(("BEFORE_WITH","WITH_EXCEPT_START")) and node_match(ast.With): + return + + if inst_match(("STORE_NAME", "STORE_GLOBAL"), argval="__doc__") and node_match( + ast.Constant + ): + # store docstrings + return + + if ( + inst_match(("STORE_NAME", "STORE_FAST", "STORE_GLOBAL", "STORE_DEREF")) + and node_match(ast.ExceptHandler) + and instruction.argval == mangled_name(node) + ): + # store exception in variable + return + + if ( + inst_match(("STORE_NAME", "STORE_FAST", "STORE_DEREF", "STORE_GLOBAL")) + and node_match((ast.Import, ast.ImportFrom)) + and any(mangled_name(cast(EnhancedAST, alias)) == instruction.argval for alias in cast(ast.Import, node).names) + ): + # store imported module in variable + return + + if ( + inst_match(("STORE_FAST", "STORE_DEREF", "STORE_NAME", "STORE_GLOBAL")) + and ( + node_match((ast.FunctionDef, ast.ClassDef, ast.AsyncFunctionDef)) + or node_match( + ast.Name, + ctx=ast.Store, + ) + ) + and instruction.argval == mangled_name(node) + ): + return + + if False: + # TODO: match expressions are not supported for now + if inst_match(("STORE_FAST", "STORE_NAME")) and node_match( + ast.MatchAs, name=instruction.argval + ): + return + + if inst_match("COMPARE_OP", argval="==") and node_match(ast.MatchSequence): + return + + if inst_match("COMPARE_OP", argval="==") and node_match(ast.MatchValue): + return + + if inst_match("BINARY_OP"): + arg=instruction.argrepr.removesuffix("=") + + if arg!="[]" and node_match( ast.AugAssign, op=op_type_map[arg]): + # a+=5 + return + + if node_match(ast.Attribute, ctx=ast.Del) and inst_match( + "DELETE_ATTR", argval=mangled_name(node) + ): + return + + if inst_match( + ( + "JUMP_IF_TRUE_OR_POP", + "JUMP_IF_FALSE_OR_POP", + "POP_JUMP_IF_TRUE", + "POP_JUMP_IF_FALSE", + ) + ) and node_match(ast.BoolOp): + # and/or short circuit + return + + if inst_match("DELETE_SUBSCR") and node_match(ast.Subscript, ctx=ast.Del): + return + + if ( + node_match(ast.Name, ctx=ast.Load) + or ( + node_match(ast.Name, ctx=ast.Store) + and isinstance(node.parent, ast.AugAssign) + ) + ) and inst_match( + ( + "LOAD_NAME", + "LOAD_FAST", + "LOAD_FAST_CHECK", + "LOAD_FAST_BORROW", + "LOAD_GLOBAL", + "LOAD_DEREF", + "LOAD_FROM_DICT_OR_DEREF", + "LOAD_FAST_BORROW_LOAD_FAST_BORROW", + ), + ) and ( + mangled_name(node) in instruction.argval if isinstance(instruction.argval,tuple) + else instruction.argval == mangled_name(node) + ): + return + + if node_match(ast.Name, ctx=ast.Del) and inst_match( + ("DELETE_NAME", "DELETE_GLOBAL", "DELETE_DEREF"), argval=mangled_name(node) + ): + return + + if node_match(ast.Constant) and inst_match( + ("LOAD_CONST","LOAD_SMALL_INT"), argval=cast(ast.Constant, node).value + ): + return + + if node_match( + (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp, ast.For) + ) and inst_match(("GET_ITER", "FOR_ITER")): + return + + if sys.version_info >= (3, 12): + if node_match(ast.UnaryOp, op=ast.UAdd) and inst_match( + "CALL_INTRINSIC_1", argrepr="INTRINSIC_UNARY_POSITIVE" + ): + return + + if node_match(ast.Subscript) and inst_match("BINARY_SLICE"): + return + + if node_match(ast.ImportFrom) and inst_match( + "CALL_INTRINSIC_1", argrepr="INTRINSIC_IMPORT_STAR" + ): + return + + if ( + node_match(ast.Yield) or isinstance(node.parent, ast.GeneratorExp) + ) and inst_match("CALL_INTRINSIC_1", argrepr="INTRINSIC_ASYNC_GEN_WRAP"): + return + + if node_match(ast.Name) and inst_match("LOAD_DEREF",argval="__classdict__"): + return + + if node_match(ast.TypeVar) and ( + inst_match("CALL_INTRINSIC_1", argrepr="INTRINSIC_TYPEVAR") + or inst_match( + "CALL_INTRINSIC_2", argrepr="INTRINSIC_TYPEVAR_WITH_BOUND" + ) + or inst_match( + "CALL_INTRINSIC_2", argrepr="INTRINSIC_TYPEVAR_WITH_CONSTRAINTS" + ) + or inst_match(("STORE_FAST", "STORE_DEREF"), argrepr=mangled_name(node)) + ): + return + + if node_match(ast.TypeVarTuple) and ( + inst_match("CALL_INTRINSIC_1", argrepr="INTRINSIC_TYPEVARTUPLE") + or inst_match(("STORE_FAST", "STORE_DEREF"), argrepr=node.name) + ): + return + + if node_match(ast.ParamSpec) and ( + inst_match("CALL_INTRINSIC_1", argrepr="INTRINSIC_PARAMSPEC") + + or inst_match(("STORE_FAST", "STORE_DEREF"), argrepr=node.name)): + return + + + if node_match(ast.TypeAlias): + if( + inst_match("CALL_INTRINSIC_1", argrepr="INTRINSIC_TYPEALIAS") + or inst_match( + ("STORE_NAME", "STORE_FAST", "STORE_DEREF","STORE_GLOBAL"), argrepr=node.name.id + ) + or inst_match("CALL") + ): + return + + + if node_match(ast.ClassDef) and node.type_params: + if inst_match( + ("STORE_DEREF", "LOAD_DEREF", "LOAD_FROM_DICT_OR_DEREF"), + argrepr=".type_params", + ): + return + + if inst_match(("STORE_FAST", "LOAD_FAST"), argrepr=".generic_base"): + return + + if inst_match( + "CALL_INTRINSIC_1", argrepr="INTRINSIC_SUBSCRIPT_GENERIC" + ): + return + + if inst_match("LOAD_DEREF",argval="__classdict__"): + return + + if node_match((ast.FunctionDef,ast.AsyncFunctionDef)) and node.type_params: + if inst_match("CALL"): + return + + if inst_match( + "CALL_INTRINSIC_2", argrepr="INTRINSIC_SET_FUNCTION_TYPE_PARAMS" + ): + return + + if inst_match("LOAD_FAST",argval=".defaults"): + return + + if inst_match("LOAD_FAST",argval=".kwdefaults"): + return + + if sys.version_info >= (3, 14): + if inst_match("LOAD_FAST_BORROW_LOAD_FAST_BORROW",argval=(".defaults",".kwdefaults")): + return + + if inst_match("STORE_NAME", argval="__classdictcell__"): + # this is a general thing + return + + + # f-strings + + if node_match(ast.JoinedStr) and ( + inst_match("LOAD_ATTR", argval="join") + or inst_match(("LIST_APPEND", "CALL")) + ): + return + + if node_match(ast.FormattedValue) and inst_match("FORMAT_VALUE"): + return + + if sys.version_info >= (3, 13): + + if inst_match("NOP"): + return + + if inst_match("TO_BOOL") and node_match(ast.BoolOp): + return + + if inst_match("CALL_KW") and node_match((ast.Call, ast.ClassDef)): + return + + if inst_match("LOAD_FAST", argval=".type_params"): + return + + if inst_match("LOAD_FAST", argval="__classdict__"): + return + + if inst_match(("LOAD_FAST","LOAD_FAST_BORROW")) and node_match( + ( + ast.FunctionDef, + ast.ClassDef, + ast.TypeAlias, + ast.TypeVar, + ast.Lambda, + ast.AsyncFunctionDef, + ) + ): + # These are loads for closure variables. + # It is difficult to check that this is actually closure variable, see: + # https://github.com/alexmojaki/executing/pull/80#discussion_r1716027317 + return + + if ( + inst_match("LOAD_FAST") + and node_match(ast.TypeAlias) + and node.name.id == instruction.argval + ): + return + + if inst_match("STORE_NAME",argval="__static_attributes__"): + # the node is the first node in the body + return + + if inst_match(("LOAD_FAST","LOAD_FAST_BORROW")) and isinstance(node.parent,ast.TypeVar): + return + + if inst_match("CALL_INTRINSIC_2",argrepr="INTRINSIC_SET_TYPEPARAM_DEFAULT") and node_match((ast.TypeVar,ast.ParamSpec,ast.TypeVarTuple)): + return + + if sys.version_info >= (3, 14): + if inst_match("BINARY_OP",argrepr="[]") and node_match(ast.Subscript): + return + if inst_match("LOAD_FAST_BORROW", argval="__classdict__"): + return + if inst_match(("STORE_NAME","LOAD_NAME"), argval="__conditional_annotations__"): + return + + if inst_match("LOAD_FAST_BORROW_LOAD_FAST_BORROW") and node_match(ast.Name) and node.id in instruction.argval: + return + + + # old verifier + + typ: Type = type(None) + op_type: Type = type(None) + + if op_name.startswith(("BINARY_SUBSCR", "SLICE+")): + typ = ast.Subscript + ctx = ast.Load + elif op_name.startswith("BINARY_"): + typ = ast.BinOp + op_type = op_type_map[instruction.argrepr] + extra_filter = lambda e: isinstance(cast(ast.BinOp, e).op, op_type) + elif op_name.startswith("UNARY_"): + typ = ast.UnaryOp + op_type = dict( + UNARY_POSITIVE=ast.UAdd, + UNARY_NEGATIVE=ast.USub, + UNARY_NOT=ast.Not, + UNARY_INVERT=ast.Invert, + )[op_name] + extra_filter = lambda e: isinstance(cast(ast.UnaryOp, e).op, op_type) + elif op_name in ("LOAD_ATTR", "LOAD_METHOD", "LOOKUP_METHOD","LOAD_SUPER_ATTR"): + typ = ast.Attribute + ctx = ast.Load + extra_filter = lambda e: mangled_name(e) == instruction.argval + elif op_name in ( + "LOAD_NAME", + "LOAD_GLOBAL", + "LOAD_FAST", + "LOAD_DEREF", + "LOAD_CLASSDEREF", + ): + typ = ast.Name + ctx = ast.Load + extra_filter = lambda e: cast(ast.Name, e).id == instruction.argval + elif op_name in ("COMPARE_OP", "IS_OP", "CONTAINS_OP"): + typ = ast.Compare + extra_filter = lambda e: len(cast(ast.Compare, e).ops) == 1 + elif op_name.startswith(("STORE_SLICE", "STORE_SUBSCR")): + ctx = ast.Store + typ = ast.Subscript + elif op_name.startswith("STORE_ATTR"): + ctx = ast.Store + typ = ast.Attribute + extra_filter = lambda e: mangled_name(e) == instruction.argval + + node_ctx = getattr(node, "ctx", None) + + ctx_match = ( + ctx is not type(None) + or not hasattr(node, "ctx") + or isinstance(node_ctx, ctx) + ) + + # check for old verifier + if isinstance(node, typ) and ctx_match and extra_filter(node): + return + + # generate error + + title = "ast.%s is not created from %s" % ( + type(node).__name__, + instruction.opname, + ) + + raise VerifierFailure(title, node, instruction) + + def instruction(self, index: int) -> Optional[dis.Instruction]: + return self.bc_dict.get(index,None) + + def instruction_before( + self, instruction: dis.Instruction + ) -> Optional[dis.Instruction]: + return self.bc_dict.get(instruction.offset - 2, None) + + def opname(self, index: int) -> str: + i=self.instruction(index) + if i is None: + return "CACHE" + return i.opname + + extra_node_types=() + if sys.version_info >= (3,12): + extra_node_types = (ast.type_param,) + + def find_node( + self, + index: int, + match_positions: Sequence[str] = ( + "lineno", + "end_lineno", + "col_offset", + "end_col_offset", + ), + typ: tuple[Type, ...] = ( + ast.expr, + ast.stmt, + ast.excepthandler, + ast.pattern, + *extra_node_types, + ), + ) -> EnhancedAST: + instruction = self.instruction(index) + assert instruction is not None + + position = instruction.positions + assert position is not None and position.lineno is not None + + return only( + cast(EnhancedAST, node) + for node in self.source._nodes_by_line[position.lineno] + if isinstance(node, typ) + if not isinstance(node, ast.Expr) + # matchvalue.value has the same positions as matchvalue themself, so we exclude ast.MatchValue + if not isinstance(node, ast.MatchValue) + if all( + getattr(position, attr) == getattr(node, attr) + for attr in match_positions + ) + ) diff --git a/lib/python3.12/site-packages/executing/_pytest_utils.py b/lib/python3.12/site-packages/executing/_pytest_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fab8693baf20570fd34b8d4bcb9242ba8878b8fb --- /dev/null +++ b/lib/python3.12/site-packages/executing/_pytest_utils.py @@ -0,0 +1,16 @@ +import sys + + + +def is_pytest_compatible() -> bool: + """ returns true if executing can be used for expressions inside assert statements which are rewritten by pytest + """ + if sys.version_info < (3, 11): + return False + + try: + import pytest + except ImportError: + return False + + return pytest.version_tuple >= (8, 3, 4) diff --git a/lib/python3.12/site-packages/executing/_utils.py b/lib/python3.12/site-packages/executing/_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..42511844eb18b0cca6c17574e2b16b199ecc5c16 --- /dev/null +++ b/lib/python3.12/site-packages/executing/_utils.py @@ -0,0 +1,139 @@ + +import ast +import sys +import dis +from typing import cast, Any,Iterator +import types + + + +def assert_(condition, message=""): + # type: (Any, str) -> None + """ + Like an assert statement, but unaffected by -O + :param condition: value that is expected to be truthy + :type message: Any + """ + if not condition: + raise AssertionError(str(message)) + + +if sys.version_info >= (3, 4): + # noinspection PyUnresolvedReferences + _get_instructions = dis.get_instructions + from dis import Instruction as _Instruction + + class Instruction(_Instruction): + lineno = None # type: int +else: + from collections import namedtuple + + class Instruction(namedtuple('Instruction', 'offset argval opname starts_line')): + lineno = None # type: int + + from dis import HAVE_ARGUMENT, EXTENDED_ARG, hasconst, opname, findlinestarts, hasname + + # Based on dis.disassemble from 2.7 + # Left as similar as possible for easy diff + + def _get_instructions(co): + # type: (types.CodeType) -> Iterator[Instruction] + code = co.co_code + linestarts = dict(findlinestarts(co)) + n = len(code) + i = 0 + extended_arg = 0 + while i < n: + offset = i + c = code[i] + op = ord(c) + lineno = linestarts.get(i) + argval = None + i = i + 1 + if op >= HAVE_ARGUMENT: + oparg = ord(code[i]) + ord(code[i + 1]) * 256 + extended_arg + extended_arg = 0 + i = i + 2 + if op == EXTENDED_ARG: + extended_arg = oparg * 65536 + + if op in hasconst: + argval = co.co_consts[oparg] + elif op in hasname: + argval = co.co_names[oparg] + elif opname[op] == 'LOAD_FAST': + argval = co.co_varnames[oparg] + yield Instruction(offset, argval, opname[op], lineno) + +def get_instructions(co): + # type: (types.CodeType) -> Iterator[EnhancedInstruction] + lineno = co.co_firstlineno + for inst in _get_instructions(co): + inst = cast(EnhancedInstruction, inst) + lineno = inst.starts_line or lineno + assert_(lineno) + inst.lineno = lineno + yield inst + + +# Type class used to expand out the definition of AST to include fields added by this library +# It's not actually used for anything other than type checking though! +class EnhancedAST(ast.AST): + parent = None # type: EnhancedAST + +# Type class used to expand out the definition of AST to include fields added by this library +# It's not actually used for anything other than type checking though! +class EnhancedInstruction(Instruction): + _copied = None # type: bool + + + + + +def mangled_name(node): + # type: (EnhancedAST) -> str + """ + + Parameters: + node: the node which should be mangled + name: the name of the node + + Returns: + The mangled name of `node` + """ + + function_class_types=(ast.FunctionDef, ast.ClassDef, ast.AsyncFunctionDef) + + if isinstance(node, ast.Attribute): + name = node.attr + elif isinstance(node, ast.Name): + name = node.id + elif isinstance(node, (ast.alias)): + name = node.asname or node.name.split(".")[0] + elif isinstance(node, function_class_types): + name = node.name + elif isinstance(node, ast.ExceptHandler): + assert node.name + name = node.name + elif sys.version_info >= (3,12) and isinstance(node,ast.TypeVar): + name=node.name + else: + raise TypeError("no node to mangle") + + if name.startswith("__") and not name.endswith("__"): + + parent,child=node.parent,node + + while not (isinstance(parent,ast.ClassDef) and child not in parent.bases): + if not hasattr(parent,"parent"): + break # pragma: no mutate + + parent,child=parent.parent,parent + else: + class_name=parent.name.lstrip("_") + if class_name!="" and child not in parent.decorator_list: + return "_" + class_name + name + + + + return name diff --git a/lib/python3.12/site-packages/executing/executing.py b/lib/python3.12/site-packages/executing/executing.py new file mode 100644 index 0000000000000000000000000000000000000000..dd1e1d798551d3e6caa142d650668bd91abed48f --- /dev/null +++ b/lib/python3.12/site-packages/executing/executing.py @@ -0,0 +1,1108 @@ +""" +MIT License + +Copyright (c) 2021 Alex Hall + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import __future__ +import ast +import dis +import inspect +import io +import linecache +import re +import sys +import types +from collections import defaultdict +from copy import deepcopy +from functools import lru_cache +from itertools import islice +from itertools import zip_longest +from operator import attrgetter +from pathlib import Path +from threading import RLock +from tokenize import detect_encoding +from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, Iterator, List, Optional, Sequence, Set, Sized, Tuple, Type, TypeVar, Union, cast +from ._utils import mangled_name,assert_, EnhancedAST,EnhancedInstruction,Instruction,get_instructions + +if TYPE_CHECKING: # pragma: no cover + from asttokens import ASTTokens, ASTText + from asttokens.asttokens import ASTTextBase + + +function_node_types = (ast.FunctionDef, ast.AsyncFunctionDef) # type: Tuple[Type, ...] + +cache = lru_cache(maxsize=None) + +TESTING = 0 + +class NotOneValueFound(Exception): + def __init__(self,msg,values=[]): + # type: (str, Sequence) -> None + self.values=values + super(NotOneValueFound,self).__init__(msg) + +T = TypeVar('T') + + +def only(it): + # type: (Iterable[T]) -> T + if isinstance(it, Sized): + if len(it) != 1: + raise NotOneValueFound('Expected one value, found %s' % len(it)) + # noinspection PyTypeChecker + return list(it)[0] + + lst = tuple(islice(it, 2)) + if len(lst) == 0: + raise NotOneValueFound('Expected one value, found 0') + if len(lst) > 1: + raise NotOneValueFound('Expected one value, found several',lst) + return lst[0] + + +class Source(object): + """ + The source code of a single file and associated metadata. + + The main method of interest is the classmethod `executing(frame)`. + + If you want an instance of this class, don't construct it. + Ideally use the classmethod `for_frame(frame)`. + If you don't have a frame, use `for_filename(filename [, module_globals])`. + These methods cache instances by filename, so at most one instance exists per filename. + + Attributes: + - filename + - text + - lines + - tree: AST parsed from text, or None if text is not valid Python + All nodes in the tree have an extra `parent` attribute + + Other methods of interest: + - statements_at_line + - asttokens + - code_qualname + """ + + def __init__(self, filename, lines): + # type: (str, Sequence[str]) -> None + """ + Don't call this constructor, see the class docstring. + """ + + self.filename = filename + self.text = ''.join(lines) + self.lines = [line.rstrip('\r\n') for line in lines] + + self._nodes_by_line = defaultdict(list) + self.tree = None + self._qualnames = {} + self._asttokens = None # type: Optional[ASTTokens] + self._asttext = None # type: Optional[ASTText] + + try: + self.tree = ast.parse(self.text, filename=filename) + except (SyntaxError, ValueError): + pass + else: + for node in ast.walk(self.tree): + for child in ast.iter_child_nodes(node): + cast(EnhancedAST, child).parent = cast(EnhancedAST, node) + for lineno in node_linenos(node): + self._nodes_by_line[lineno].append(node) + + visitor = QualnameVisitor() + visitor.visit(self.tree) + self._qualnames = visitor.qualnames + + @classmethod + def for_frame(cls, frame, use_cache=True): + # type: (types.FrameType, bool) -> "Source" + """ + Returns the `Source` object corresponding to the file the frame is executing in. + """ + return cls.for_filename(frame.f_code.co_filename, frame.f_globals or {}, use_cache) + + @classmethod + def for_filename( + cls, + filename, + module_globals=None, + use_cache=True, # noqa no longer used + ): + # type: (Union[str, Path], Optional[Dict[str, Any]], bool) -> "Source" + if isinstance(filename, Path): + filename = str(filename) + + def get_lines(): + # type: () -> List[str] + return linecache.getlines(cast(str, filename), module_globals) + + # Save the current linecache entry, then ensure the cache is up to date. + entry = linecache.cache.get(filename) # type: ignore[attr-defined] + linecache.checkcache(filename) + lines = get_lines() + if entry is not None and not lines: + # There was an entry, checkcache removed it, and nothing replaced it. + # This means the file wasn't simply changed (because the `lines` wouldn't be empty) + # but rather the file was found not to exist, probably because `filename` was fake. + # Restore the original entry so that we still have something. + linecache.cache[filename] = entry # type: ignore[attr-defined] + lines = get_lines() + + return cls._for_filename_and_lines(filename, tuple(lines)) + + @classmethod + def _for_filename_and_lines(cls, filename, lines): + # type: (str, Sequence[str]) -> "Source" + source_cache = cls._class_local('__source_cache_with_lines', {}) # type: Dict[Tuple[str, Sequence[str]], Source] + try: + return source_cache[(filename, lines)] + except KeyError: + pass + + result = source_cache[(filename, lines)] = cls(filename, lines) + return result + + @classmethod + def lazycache(cls, frame): + # type: (types.FrameType) -> None + linecache.lazycache(frame.f_code.co_filename, frame.f_globals) + + @classmethod + def executing(cls, frame_or_tb): + # type: (Union[types.TracebackType, types.FrameType]) -> "Executing" + """ + Returns an `Executing` object representing the operation + currently executing in the given frame or traceback object. + """ + if isinstance(frame_or_tb, types.TracebackType): + # https://docs.python.org/3/reference/datamodel.html#traceback-objects + # "tb_lineno gives the line number where the exception occurred; + # tb_lasti indicates the precise instruction. + # The line number and last instruction in the traceback may differ + # from the line number of its frame object + # if the exception occurred in a try statement with no matching except clause + # or with a finally clause." + tb = frame_or_tb + frame = tb.tb_frame + lineno = tb.tb_lineno + lasti = tb.tb_lasti + else: + frame = frame_or_tb + lineno = frame.f_lineno + lasti = frame.f_lasti + + + + code = frame.f_code + key = (code, id(code), lasti) + executing_cache = cls._class_local('__executing_cache', {}) # type: Dict[Tuple[types.CodeType, int, int], Any] + + args = executing_cache.get(key) + if not args: + node = stmts = decorator = None + source = cls.for_frame(frame) + tree = source.tree + if tree: + try: + stmts = source.statements_at_line(lineno) + if stmts: + if is_ipython_cell_code(code): + decorator, node = find_node_ipython(frame, lasti, stmts, source) + else: + node_finder = NodeFinder(frame, stmts, tree, lasti, source) + node = node_finder.result + decorator = node_finder.decorator + + if node: + new_stmts = {statement_containing_node(node)} + assert_(new_stmts <= stmts) + stmts = new_stmts + except Exception: + if TESTING: + raise + + executing_cache[key] = args = source, node, stmts, decorator + + return Executing(frame, *args) + + @classmethod + def _class_local(cls, name, default): + # type: (str, T) -> T + """ + Returns an attribute directly associated with this class + (as opposed to subclasses), setting default if necessary + """ + # classes have a mappingproxy preventing us from using setdefault + result = cls.__dict__.get(name, default) + setattr(cls, name, result) + return result + + @cache + def statements_at_line(self, lineno): + # type: (int) -> Set[EnhancedAST] + """ + Returns the statement nodes overlapping the given line. + + Returns at most one statement unless semicolons are present. + + If the `text` attribute is not valid python, meaning + `tree` is None, returns an empty set. + + Otherwise, `Source.for_frame(frame).statements_at_line(frame.f_lineno)` + should return at least one statement. + """ + + return { + statement_containing_node(node) + for node in + self._nodes_by_line[lineno] + } + + def asttext(self): + # type: () -> ASTText + """ + Returns an ASTText object for getting the source of specific AST nodes. + + See http://asttokens.readthedocs.io/en/latest/api-index.html + """ + from asttokens import ASTText # must be installed separately + + if self._asttext is None: + self._asttext = ASTText(self.text, tree=self.tree, filename=self.filename) + + return self._asttext + + def asttokens(self): + # type: () -> ASTTokens + """ + Returns an ASTTokens object for getting the source of specific AST nodes. + + See http://asttokens.readthedocs.io/en/latest/api-index.html + """ + import asttokens # must be installed separately + + if self._asttokens is None: + if hasattr(asttokens, 'ASTText'): + self._asttokens = self.asttext().asttokens + else: # pragma: no cover + self._asttokens = asttokens.ASTTokens(self.text, tree=self.tree, filename=self.filename) + return self._asttokens + + def _asttext_base(self): + # type: () -> ASTTextBase + import asttokens # must be installed separately + + if hasattr(asttokens, 'ASTText'): + return self.asttext() + else: # pragma: no cover + return self.asttokens() + + @staticmethod + def decode_source(source): + # type: (Union[str, bytes]) -> str + if isinstance(source, bytes): + encoding = Source.detect_encoding(source) + return source.decode(encoding) + else: + return source + + @staticmethod + def detect_encoding(source): + # type: (bytes) -> str + return detect_encoding(io.BytesIO(source).readline)[0] + + def code_qualname(self, code): + # type: (types.CodeType) -> str + """ + Imitates the __qualname__ attribute of functions for code objects. + Given: + + - A function `func` + - A frame `frame` for an execution of `func`, meaning: + `frame.f_code is func.__code__` + + `Source.for_frame(frame).code_qualname(frame.f_code)` + will be equal to `func.__qualname__`*. Works for Python 2 as well, + where of course no `__qualname__` attribute exists. + + Falls back to `code.co_name` if there is no appropriate qualname. + + Based on https://github.com/wbolster/qualname + + (* unless `func` is a lambda + nested inside another lambda on the same line, in which case + the outer lambda's qualname will be returned for the codes + of both lambdas) + """ + assert_(code.co_filename == self.filename) + return self._qualnames.get((code.co_name, code.co_firstlineno), code.co_name) + + +class Executing(object): + """ + Information about the operation a frame is currently executing. + + Generally you will just want `node`, which is the AST node being executed, + or None if it's unknown. + + If a decorator is currently being called, then: + - `node` is a function or class definition + - `decorator` is the expression in `node.decorator_list` being called + - `statements == {node}` + """ + + def __init__(self, frame, source, node, stmts, decorator): + # type: (types.FrameType, Source, EnhancedAST, Set[ast.stmt], Optional[EnhancedAST]) -> None + self.frame = frame + self.source = source + self.node = node + self.statements = stmts + self.decorator = decorator + + def code_qualname(self): + # type: () -> str + return self.source.code_qualname(self.frame.f_code) + + def text(self): + # type: () -> str + return self.source._asttext_base().get_text(self.node) + + def text_range(self): + # type: () -> Tuple[int, int] + return self.source._asttext_base().get_text_range(self.node) + + +class QualnameVisitor(ast.NodeVisitor): + def __init__(self): + # type: () -> None + super(QualnameVisitor, self).__init__() + self.stack = [] # type: List[str] + self.qualnames = {} # type: Dict[Tuple[str, int], str] + + def add_qualname(self, node, name=None): + # type: (ast.AST, Optional[str]) -> None + name = name or node.name # type: ignore[attr-defined] + self.stack.append(name) + if getattr(node, 'decorator_list', ()): + lineno = node.decorator_list[0].lineno # type: ignore[attr-defined] + else: + lineno = node.lineno # type: ignore[attr-defined] + self.qualnames.setdefault((name, lineno), ".".join(self.stack)) + + def visit_FunctionDef(self, node, name=None): + # type: (ast.AST, Optional[str]) -> None + assert isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)), node + self.add_qualname(node, name) + self.stack.append('') + children = [] # type: Sequence[ast.AST] + if isinstance(node, ast.Lambda): + children = [node.body] + else: + children = node.body + for child in children: + self.visit(child) + self.stack.pop() + self.stack.pop() + + # Find lambdas in the function definition outside the body, + # e.g. decorators or default arguments + # Based on iter_child_nodes + for field, child in ast.iter_fields(node): + if field == 'body': + continue + if isinstance(child, ast.AST): + self.visit(child) + elif isinstance(child, list): + for grandchild in child: + if isinstance(grandchild, ast.AST): + self.visit(grandchild) + + visit_AsyncFunctionDef = visit_FunctionDef + + def visit_Lambda(self, node): + # type: (ast.AST) -> None + assert isinstance(node, ast.Lambda) + self.visit_FunctionDef(node, '') + + def visit_ClassDef(self, node): + # type: (ast.AST) -> None + assert isinstance(node, ast.ClassDef) + self.add_qualname(node) + self.generic_visit(node) + self.stack.pop() + + + + + +future_flags = sum( + getattr(__future__, fname).compiler_flag for fname in __future__.all_feature_names +) + + +def compile_similar_to(source, matching_code): + # type: (ast.Module, types.CodeType) -> Any + return compile( + source, + matching_code.co_filename, + 'exec', + flags=future_flags & matching_code.co_flags, + dont_inherit=True, + ) + + +sentinel = 'io8urthglkjdghvljusketgIYRFYUVGHFRTBGVHKGF78678957647698' + +def is_rewritten_by_pytest(code): + # type: (types.CodeType) -> bool + return any( + bc.opname != "LOAD_CONST" and isinstance(bc.argval,str) and bc.argval.startswith("@py") + for bc in get_instructions(code) + ) + + +class SentinelNodeFinder(object): + result = None # type: EnhancedAST + + def __init__(self, frame, stmts, tree, lasti, source): + # type: (types.FrameType, Set[EnhancedAST], ast.Module, int, Source) -> None + assert_(stmts) + self.frame = frame + self.tree = tree + self.code = code = frame.f_code + self.is_pytest = is_rewritten_by_pytest(code) + + if self.is_pytest: + self.ignore_linenos = frozenset(assert_linenos(tree)) + else: + self.ignore_linenos = frozenset() + + self.decorator = None + + self.instruction = instruction = self.get_actual_current_instruction(lasti) + op_name = instruction.opname + extra_filter = lambda e: True # type: Callable[[Any], bool] + ctx = type(None) # type: Type + + typ = type(None) # type: Type + if op_name.startswith('CALL_'): + typ = ast.Call + elif op_name.startswith(('BINARY_SUBSCR', 'SLICE+')): + typ = ast.Subscript + ctx = ast.Load + elif op_name.startswith('BINARY_'): + typ = ast.BinOp + op_type = dict( + BINARY_POWER=ast.Pow, + BINARY_MULTIPLY=ast.Mult, + BINARY_MATRIX_MULTIPLY=getattr(ast, "MatMult", ()), + BINARY_FLOOR_DIVIDE=ast.FloorDiv, + BINARY_TRUE_DIVIDE=ast.Div, + BINARY_MODULO=ast.Mod, + BINARY_ADD=ast.Add, + BINARY_SUBTRACT=ast.Sub, + BINARY_LSHIFT=ast.LShift, + BINARY_RSHIFT=ast.RShift, + BINARY_AND=ast.BitAnd, + BINARY_XOR=ast.BitXor, + BINARY_OR=ast.BitOr, + )[op_name] + extra_filter = lambda e: isinstance(e.op, op_type) + elif op_name.startswith('UNARY_'): + typ = ast.UnaryOp + op_type = dict( + UNARY_POSITIVE=ast.UAdd, + UNARY_NEGATIVE=ast.USub, + UNARY_NOT=ast.Not, + UNARY_INVERT=ast.Invert, + )[op_name] + extra_filter = lambda e: isinstance(e.op, op_type) + elif op_name in ('LOAD_ATTR', 'LOAD_METHOD', 'LOOKUP_METHOD'): + typ = ast.Attribute + ctx = ast.Load + extra_filter = lambda e:mangled_name(e) == instruction.argval + elif op_name in ('LOAD_NAME', 'LOAD_GLOBAL', 'LOAD_FAST', 'LOAD_DEREF', 'LOAD_CLASSDEREF'): + typ = ast.Name + ctx = ast.Load + extra_filter = lambda e:mangled_name(e) == instruction.argval + elif op_name in ('COMPARE_OP', 'IS_OP', 'CONTAINS_OP'): + typ = ast.Compare + extra_filter = lambda e: len(e.ops) == 1 + elif op_name.startswith(('STORE_SLICE', 'STORE_SUBSCR')): + ctx = ast.Store + typ = ast.Subscript + elif op_name.startswith('STORE_ATTR'): + ctx = ast.Store + typ = ast.Attribute + extra_filter = lambda e:mangled_name(e) == instruction.argval + else: + raise RuntimeError(op_name) + + + with lock: + exprs = { + cast(EnhancedAST, node) + for stmt in stmts + for node in ast.walk(stmt) + if isinstance(node, typ) + if isinstance(getattr(node, "ctx", None), ctx) + if extra_filter(node) + if statement_containing_node(node) == stmt + } + + if ctx == ast.Store: + # No special bytecode tricks here. + # We can handle multiple assigned attributes with different names, + # but only one assigned subscript. + self.result = only(exprs) + return + + matching = list(self.matching_nodes(exprs)) + if not matching and typ == ast.Call: + self.find_decorator(stmts) + else: + self.result = only(matching) + + def find_decorator(self, stmts): + # type: (Union[List[EnhancedAST], Set[EnhancedAST]]) -> None + stmt = only(stmts) + assert_(isinstance(stmt, (ast.ClassDef, function_node_types))) + decorators = stmt.decorator_list # type: ignore[attr-defined] + assert_(decorators) + line_instructions = [ + inst + for inst in self.clean_instructions(self.code) + if inst.lineno == self.frame.f_lineno + ] + last_decorator_instruction_index = [ + i + for i, inst in enumerate(line_instructions) + if inst.opname == "CALL_FUNCTION" + ][-1] + assert_( + line_instructions[last_decorator_instruction_index + 1].opname.startswith( + "STORE_" + ) + ) + decorator_instructions = line_instructions[ + last_decorator_instruction_index + - len(decorators) + + 1 : last_decorator_instruction_index + + 1 + ] + assert_({inst.opname for inst in decorator_instructions} == {"CALL_FUNCTION"}) + decorator_index = decorator_instructions.index(self.instruction) + decorator = decorators[::-1][decorator_index] + self.decorator = decorator + self.result = stmt + + def clean_instructions(self, code): + # type: (types.CodeType) -> List[EnhancedInstruction] + return [ + inst + for inst in get_instructions(code) + if inst.opname not in ("EXTENDED_ARG", "NOP") + if inst.lineno not in self.ignore_linenos + ] + + def get_original_clean_instructions(self): + # type: () -> List[EnhancedInstruction] + result = self.clean_instructions(self.code) + + # pypy sometimes (when is not clear) + # inserts JUMP_IF_NOT_DEBUG instructions in bytecode + # If they're not present in our compiled instructions, + # ignore them in the original bytecode + if not any( + inst.opname == "JUMP_IF_NOT_DEBUG" + for inst in self.compile_instructions() + ): + result = [ + inst for inst in result + if inst.opname != "JUMP_IF_NOT_DEBUG" + ] + + return result + + def matching_nodes(self, exprs): + # type: (Set[EnhancedAST]) -> Iterator[EnhancedAST] + original_instructions = self.get_original_clean_instructions() + original_index = only( + i + for i, inst in enumerate(original_instructions) + if inst == self.instruction + ) + for expr_index, expr in enumerate(exprs): + setter = get_setter(expr) + assert setter is not None + # noinspection PyArgumentList + replacement = ast.BinOp( + left=expr, + op=ast.Pow(), + right=ast.Str(s=sentinel), + ) + ast.fix_missing_locations(replacement) + setter(replacement) + try: + instructions = self.compile_instructions() + finally: + setter(expr) + + if sys.version_info >= (3, 10): + try: + handle_jumps(instructions, original_instructions) + except Exception: + # Give other candidates a chance + if TESTING or expr_index < len(exprs) - 1: + continue + raise + + indices = [ + i + for i, instruction in enumerate(instructions) + if instruction.argval == sentinel + ] + + # There can be several indices when the bytecode is duplicated, + # as happens in a finally block in 3.9+ + # First we remove the opcodes caused by our modifications + for index_num, sentinel_index in enumerate(indices): + # Adjustment for removing sentinel instructions below + # in past iterations + sentinel_index -= index_num * 2 + + assert_(instructions.pop(sentinel_index).opname == 'LOAD_CONST') + assert_(instructions.pop(sentinel_index).opname == 'BINARY_POWER') + + # Then we see if any of the instruction indices match + for index_num, sentinel_index in enumerate(indices): + sentinel_index -= index_num * 2 + new_index = sentinel_index - 1 + + if new_index != original_index: + continue + + original_inst = original_instructions[original_index] + new_inst = instructions[new_index] + + # In Python 3.9+, changing 'not x in y' to 'not sentinel_transformation(x in y)' + # changes a CONTAINS_OP(invert=1) to CONTAINS_OP(invert=0),,UNARY_NOT + if ( + original_inst.opname == new_inst.opname in ('CONTAINS_OP', 'IS_OP') + and original_inst.arg != new_inst.arg # type: ignore[attr-defined] + and ( + original_instructions[original_index + 1].opname + != instructions[new_index + 1].opname == 'UNARY_NOT' + )): + # Remove the difference for the upcoming assert + instructions.pop(new_index + 1) + + # Check that the modified instructions don't have anything unexpected + # 3.10 is a bit too weird to assert this in all cases but things still work + if sys.version_info < (3, 10): + for inst1, inst2 in zip_longest( + original_instructions, instructions + ): + assert_(inst1 and inst2 and opnames_match(inst1, inst2)) + + yield expr + + def compile_instructions(self): + # type: () -> List[EnhancedInstruction] + module_code = compile_similar_to(self.tree, self.code) + code = only(self.find_codes(module_code)) + return self.clean_instructions(code) + + def find_codes(self, root_code): + # type: (types.CodeType) -> list + checks = [ + attrgetter('co_firstlineno'), + attrgetter('co_freevars'), + attrgetter('co_cellvars'), + lambda c: is_ipython_cell_code_name(c.co_name) or c.co_name, + ] # type: List[Callable] + if not self.is_pytest: + checks += [ + attrgetter('co_names'), + attrgetter('co_varnames'), + ] + + def matches(c): + # type: (types.CodeType) -> bool + return all( + f(c) == f(self.code) + for f in checks + ) + + code_options = [] + if matches(root_code): + code_options.append(root_code) + + def finder(code): + # type: (types.CodeType) -> None + for const in code.co_consts: + if not inspect.iscode(const): + continue + + if matches(const): + code_options.append(const) + finder(const) + + finder(root_code) + return code_options + + def get_actual_current_instruction(self, lasti): + # type: (int) -> EnhancedInstruction + """ + Get the instruction corresponding to the current + frame offset, skipping EXTENDED_ARG instructions + """ + # Don't use get_original_clean_instructions + # because we need the actual instructions including + # EXTENDED_ARG + instructions = list(get_instructions(self.code)) + index = only( + i + for i, inst in enumerate(instructions) + if inst.offset == lasti + ) + + while True: + instruction = instructions[index] + if instruction.opname != "EXTENDED_ARG": + return instruction + index += 1 + + + +def non_sentinel_instructions(instructions, start): + # type: (List[EnhancedInstruction], int) -> Iterator[Tuple[int, EnhancedInstruction]] + """ + Yields (index, instruction) pairs excluding the basic + instructions introduced by the sentinel transformation + """ + skip_power = False + for i, inst in islice(enumerate(instructions), start, None): + if inst.argval == sentinel: + assert_(inst.opname == "LOAD_CONST") + skip_power = True + continue + elif skip_power: + assert_(inst.opname == "BINARY_POWER") + skip_power = False + continue + yield i, inst + + +def walk_both_instructions(original_instructions, original_start, instructions, start): + # type: (List[EnhancedInstruction], int, List[EnhancedInstruction], int) -> Iterator[Tuple[int, EnhancedInstruction, int, EnhancedInstruction]] + """ + Yields matching indices and instructions from the new and original instructions, + leaving out changes made by the sentinel transformation. + """ + original_iter = islice(enumerate(original_instructions), original_start, None) + new_iter = non_sentinel_instructions(instructions, start) + inverted_comparison = False + while True: + try: + original_i, original_inst = next(original_iter) + new_i, new_inst = next(new_iter) + except StopIteration: + return + if ( + inverted_comparison + and original_inst.opname != new_inst.opname == "UNARY_NOT" + ): + new_i, new_inst = next(new_iter) + inverted_comparison = ( + original_inst.opname == new_inst.opname in ("CONTAINS_OP", "IS_OP") + and original_inst.arg != new_inst.arg # type: ignore[attr-defined] + ) + yield original_i, original_inst, new_i, new_inst + + +def handle_jumps(instructions, original_instructions): + # type: (List[EnhancedInstruction], List[EnhancedInstruction]) -> None + """ + Transforms instructions in place until it looks more like original_instructions. + This is only needed in 3.10+ where optimisations lead to more drastic changes + after the sentinel transformation. + Replaces JUMP instructions that aren't also present in original_instructions + with the sections that they jump to until a raise or return. + In some other cases duplication found in `original_instructions` + is replicated in `instructions`. + """ + while True: + for original_i, original_inst, new_i, new_inst in walk_both_instructions( + original_instructions, 0, instructions, 0 + ): + if opnames_match(original_inst, new_inst): + continue + + if "JUMP" in new_inst.opname and "JUMP" not in original_inst.opname: + # Find where the new instruction is jumping to, ignoring + # instructions which have been copied in previous iterations + start = only( + i + for i, inst in enumerate(instructions) + if inst.offset == new_inst.argval + and not getattr(inst, "_copied", False) + ) + # Replace the jump instruction with the jumped to section of instructions + # That section may also be deleted if it's not similarly duplicated + # in original_instructions + new_instructions = handle_jump( + original_instructions, original_i, instructions, start + ) + assert new_instructions is not None + instructions[new_i : new_i + 1] = new_instructions + else: + # Extract a section of original_instructions from original_i to return/raise + orig_section = [] + for section_inst in original_instructions[original_i:]: + orig_section.append(section_inst) + if section_inst.opname in ("RETURN_VALUE", "RAISE_VARARGS"): + break + else: + # No return/raise - this is just a mismatch we can't handle + raise AssertionError + + instructions[new_i:new_i] = only(find_new_matching(orig_section, instructions)) + + # instructions has been modified, the for loop can't sensibly continue + # Restart it from the beginning, checking for other issues + break + + else: # No mismatched jumps found, we're done + return + + +def find_new_matching(orig_section, instructions): + # type: (List[EnhancedInstruction], List[EnhancedInstruction]) -> Iterator[List[EnhancedInstruction]] + """ + Yields sections of `instructions` which match `orig_section`. + The yielded sections include sentinel instructions, but these + are ignored when checking for matches. + """ + for start in range(len(instructions) - len(orig_section)): + indices, dup_section = zip( + *islice( + non_sentinel_instructions(instructions, start), + len(orig_section), + ) + ) + if len(dup_section) < len(orig_section): + return + if sections_match(orig_section, dup_section): + yield instructions[start:indices[-1] + 1] + + +def handle_jump(original_instructions, original_start, instructions, start): + # type: (List[EnhancedInstruction], int, List[EnhancedInstruction], int) -> Optional[List[EnhancedInstruction]] + """ + Returns the section of instructions starting at `start` and ending + with a RETURN_VALUE or RAISE_VARARGS instruction. + There should be a matching section in original_instructions starting at original_start. + If that section doesn't appear elsewhere in original_instructions, + then also delete the returned section of instructions. + """ + for original_j, original_inst, new_j, new_inst in walk_both_instructions( + original_instructions, original_start, instructions, start + ): + assert_(opnames_match(original_inst, new_inst)) + if original_inst.opname in ("RETURN_VALUE", "RAISE_VARARGS"): + inlined = deepcopy(instructions[start : new_j + 1]) + for inl in inlined: + inl._copied = True + orig_section = original_instructions[original_start : original_j + 1] + if not check_duplicates( + original_start, orig_section, original_instructions + ): + instructions[start : new_j + 1] = [] + return inlined + + return None + + +def check_duplicates(original_i, orig_section, original_instructions): + # type: (int, List[EnhancedInstruction], List[EnhancedInstruction]) -> bool + """ + Returns True if a section of original_instructions starting somewhere other + than original_i and matching orig_section is found, i.e. orig_section is duplicated. + """ + for dup_start in range(len(original_instructions)): + if dup_start == original_i: + continue + dup_section = original_instructions[dup_start : dup_start + len(orig_section)] + if len(dup_section) < len(orig_section): + return False + if sections_match(orig_section, dup_section): + return True + + return False + +def sections_match(orig_section, dup_section): + # type: (List[EnhancedInstruction], List[EnhancedInstruction]) -> bool + """ + Returns True if the given lists of instructions have matching linenos and opnames. + """ + return all( + ( + orig_inst.lineno == dup_inst.lineno + # POP_BLOCKs have been found to have differing linenos in innocent cases + or "POP_BLOCK" == orig_inst.opname == dup_inst.opname + ) + and opnames_match(orig_inst, dup_inst) + for orig_inst, dup_inst in zip(orig_section, dup_section) + ) + + +def opnames_match(inst1, inst2): + # type: (Instruction, Instruction) -> bool + return ( + inst1.opname == inst2.opname + or "JUMP" in inst1.opname + and "JUMP" in inst2.opname + or (inst1.opname == "PRINT_EXPR" and inst2.opname == "POP_TOP") + or ( + inst1.opname in ("LOAD_METHOD", "LOOKUP_METHOD") + and inst2.opname == "LOAD_ATTR" + ) + or (inst1.opname == "CALL_METHOD" and inst2.opname == "CALL_FUNCTION") + ) + + +def get_setter(node): + # type: (EnhancedAST) -> Optional[Callable[[ast.AST], None]] + parent = node.parent + for name, field in ast.iter_fields(parent): + if field is node: + def setter(new_node): + # type: (ast.AST) -> None + return setattr(parent, name, new_node) + return setter + elif isinstance(field, list): + for i, item in enumerate(field): + if item is node: + def setter(new_node): + # type: (ast.AST) -> None + field[i] = new_node + + return setter + return None + +lock = RLock() + + +@cache +def statement_containing_node(node): + # type: (ast.AST) -> EnhancedAST + while not isinstance(node, ast.stmt): + node = cast(EnhancedAST, node).parent + return cast(EnhancedAST, node) + + +def assert_linenos(tree): + # type: (ast.AST) -> Iterator[int] + for node in ast.walk(tree): + if ( + hasattr(node, 'parent') and + isinstance(statement_containing_node(node), ast.Assert) + ): + for lineno in node_linenos(node): + yield lineno + + +def _extract_ipython_statement(stmt): + # type: (EnhancedAST) -> ast.Module + # IPython separates each statement in a cell to be executed separately + # So NodeFinder should only compile one statement at a time or it + # will find a code mismatch. + while not isinstance(stmt.parent, ast.Module): + stmt = stmt.parent + # use `ast.parse` instead of `ast.Module` for better portability + # python3.8 changes the signature of `ast.Module` + # Inspired by https://github.com/pallets/werkzeug/pull/1552/files + tree = ast.parse("") + tree.body = [cast(ast.stmt, stmt)] + ast.copy_location(tree, stmt) + return tree + + +def is_ipython_cell_code_name(code_name): + # type: (str) -> bool + return bool(re.match(r"(|)$", code_name)) + + +def is_ipython_cell_filename(filename): + # type: (str) -> bool + return bool(re.search(r" bool + return ( + is_ipython_cell_filename(code_obj.co_filename) and + is_ipython_cell_code_name(code_obj.co_name) + ) + + +def find_node_ipython(frame, lasti, stmts, source): + # type: (types.FrameType, int, Set[EnhancedAST], Source) -> Tuple[Optional[Any], Optional[Any]] + node = decorator = None + for stmt in stmts: + tree = _extract_ipython_statement(stmt) + try: + node_finder = NodeFinder(frame, stmts, tree, lasti, source) + if (node or decorator) and (node_finder.result or node_finder.decorator): + # Found potential nodes in separate statements, + # cannot resolve ambiguity, give up here + return None, None + + node = node_finder.result + decorator = node_finder.decorator + except Exception: + pass + return decorator, node + + + +def node_linenos(node): + # type: (ast.AST) -> Iterator[int] + if hasattr(node, "lineno"): + linenos = [] # type: Sequence[int] + if hasattr(node, "end_lineno") and isinstance(node, ast.expr): + assert node.end_lineno is not None # type: ignore[attr-defined] + linenos = range(node.lineno, node.end_lineno + 1) # type: ignore[attr-defined] + else: + linenos = [node.lineno] # type: ignore[attr-defined] + for lineno in linenos: + yield lineno + + +if sys.version_info >= (3, 11): + from ._position_node_finder import PositionNodeFinder as NodeFinder +else: + NodeFinder = SentinelNodeFinder + diff --git a/lib/python3.12/site-packages/executing/py.typed b/lib/python3.12/site-packages/executing/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/executing/version.py b/lib/python3.12/site-packages/executing/version.py new file mode 100644 index 0000000000000000000000000000000000000000..1ef59ad4406053722b83ff12d60112b8cc987919 --- /dev/null +++ b/lib/python3.12/site-packages/executing/version.py @@ -0,0 +1 @@ +__version__ = '2.2.1' \ No newline at end of file diff --git a/lib/python3.12/site-packages/gitdb-4.0.12.dist-info/AUTHORS b/lib/python3.12/site-packages/gitdb-4.0.12.dist-info/AUTHORS new file mode 100644 index 0000000000000000000000000000000000000000..6c7e9b9971cca907772213fce28eb8e0162a875f --- /dev/null +++ b/lib/python3.12/site-packages/gitdb-4.0.12.dist-info/AUTHORS @@ -0,0 +1,4 @@ +Creator: Sebastian Thiel + +Contributors: + - Ram Rachum (@cool-RR) diff --git a/lib/python3.12/site-packages/gitdb-4.0.12.dist-info/INSTALLER b/lib/python3.12/site-packages/gitdb-4.0.12.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.12/site-packages/gitdb-4.0.12.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.12/site-packages/gitdb-4.0.12.dist-info/LICENSE b/lib/python3.12/site-packages/gitdb-4.0.12.dist-info/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..0d6fe8bdb9ee5bb745e2b86c9e8745676ff0ef4a --- /dev/null +++ b/lib/python3.12/site-packages/gitdb-4.0.12.dist-info/LICENSE @@ -0,0 +1,42 @@ +Copyright (C) 2010, 2011 Sebastian Thiel and contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of the GitDB project nor the names of +its contributors may be used to endorse or promote products derived +from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Additional Licenses +------------------- +The files at +gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx +and +gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack +are licensed under GNU GPL as part of the git source repository, +see http://en.wikipedia.org/wiki/Git_%28software%29 for more information. + +They are not required for the actual operation, which is why they are not found +in the distribution package. diff --git a/lib/python3.12/site-packages/gitdb-4.0.12.dist-info/METADATA b/lib/python3.12/site-packages/gitdb-4.0.12.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..a06741dcc03232d817d536ce63d15246b243f250 --- /dev/null +++ b/lib/python3.12/site-packages/gitdb-4.0.12.dist-info/METADATA @@ -0,0 +1,32 @@ +Metadata-Version: 2.1 +Name: gitdb +Version: 4.0.12 +Summary: Git Object Database +Home-page: https://github.com/gitpython-developers/gitdb +Author: Sebastian Thiel +Author-email: byronimo@gmail.com +License: BSD License +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Console +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Operating System :: POSIX +Classifier: Operating System :: Microsoft :: Windows +Classifier: Operating System :: MacOS :: MacOS X +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3 :: Only +Requires-Python: >=3.7 +License-File: LICENSE +License-File: AUTHORS +Requires-Dist: smmap<6,>=3.0.1 + +GitDB is a pure-Python git object database diff --git a/lib/python3.12/site-packages/gitdb-4.0.12.dist-info/RECORD b/lib/python3.12/site-packages/gitdb-4.0.12.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..04d43dd54f2b56f41f3b13e736cec04ff8f839f4 --- /dev/null +++ b/lib/python3.12/site-packages/gitdb-4.0.12.dist-info/RECORD @@ -0,0 +1,57 @@ +gitdb-4.0.12.dist-info/AUTHORS,sha256=aUmmuuKGJrGDzN5i-dDIbj00R1IOPcFTZDWznhEwZuM,66 +gitdb-4.0.12.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +gitdb-4.0.12.dist-info/LICENSE,sha256=79KfWWoI6IV-aOdpSlC82nKDl5LafD8EG8v_XxgAkjk,1984 +gitdb-4.0.12.dist-info/METADATA,sha256=lnDS3IZgOda3yR9SzYHH8iTNdGf3aFx8KebEaR0b89k,1230 +gitdb-4.0.12.dist-info/RECORD,, +gitdb-4.0.12.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91 +gitdb-4.0.12.dist-info/top_level.txt,sha256=ss6atT8cG4mQuAYXO6PokJ0r4Mm5cBiDbKsu2e3YHfs,6 +gitdb/__init__.py,sha256=Zfj1Uasvn9KgccHILrsHFZGTGFm4mvmEmw0trh6HOrM,551 +gitdb/__pycache__/__init__.cpython-312.pyc,, +gitdb/__pycache__/base.cpython-312.pyc,, +gitdb/__pycache__/const.cpython-312.pyc,, +gitdb/__pycache__/exc.cpython-312.pyc,, +gitdb/__pycache__/fun.cpython-312.pyc,, +gitdb/__pycache__/pack.cpython-312.pyc,, +gitdb/__pycache__/stream.cpython-312.pyc,, +gitdb/__pycache__/typ.cpython-312.pyc,, +gitdb/__pycache__/util.cpython-312.pyc,, +gitdb/base.py,sha256=krg61c_FKEvn4KAcX6pjH-hlpFfRCcM7mrki4Hnmxew,8023 +gitdb/const.py,sha256=WWmEYKNDdm3J9fxYTFT_B6-QLDSMBClbz0LSBa1D1S8,90 +gitdb/db/__init__.py,sha256=aQTZnxpfk6c76X1ubZzO9pBq4STnL6hCoGxZzXurEZ4,371 +gitdb/db/__pycache__/__init__.cpython-312.pyc,, +gitdb/db/__pycache__/base.cpython-312.pyc,, +gitdb/db/__pycache__/git.cpython-312.pyc,, +gitdb/db/__pycache__/loose.cpython-312.pyc,, +gitdb/db/__pycache__/mem.cpython-312.pyc,, +gitdb/db/__pycache__/pack.cpython-312.pyc,, +gitdb/db/__pycache__/ref.cpython-312.pyc,, +gitdb/db/base.py,sha256=_qpZ1VzwpoTcU3_-IIXmwIW0p63HXrtJ5jWNgjrLjRY,9061 +gitdb/db/git.py,sha256=b5O01eZsnzldruzEENvqiw8Q7Daz2iETrlAqH0aggdE,2666 +gitdb/db/loose.py,sha256=z7m0Cju-knikJi5UX6KWdWAJ9OOCeOE4gfYBso4XAQA,8734 +gitdb/db/mem.py,sha256=VDscunT4EtRo2GBE52lup1ki3si8enImUSKmSVq8kwc,3343 +gitdb/db/pack.py,sha256=iZPDF6rQ-kCrgbdsZRA83CL-zWsoQaZxo5rcTarjAoQ,7285 +gitdb/db/ref.py,sha256=rGOzF3cAOBW9i0Owz_Iw6c4K5BgbYFwk5btMaXGvCoc,2591 +gitdb/exc.py,sha256=B0N6I2rctCwd0Upoq2laTigbVZNOZ38nfobRcKDBtV4,1496 +gitdb/fun.py,sha256=jw3paQMgxexUn2NENFavLXfABfNvDBRfRCvv2VHkvms,23243 +gitdb/pack.py,sha256=buboXaTzXKEedNX5j1zIUIPhnLmHkDRl7Rg9ftclt14,39228 +gitdb/stream.py,sha256=0-BWcggWWZniMXGGaAe3MR6iH9DBt5ir9YtgE2_rKKw,27541 +gitdb/test/__init__.py,sha256=4cJQwIMepzCxK0hdxDegJGxNVK4dungjbPzCUJc1ZIU,204 +gitdb/test/__pycache__/__init__.cpython-312.pyc,, +gitdb/test/__pycache__/lib.cpython-312.pyc,, +gitdb/test/__pycache__/test_base.cpython-312.pyc,, +gitdb/test/__pycache__/test_example.cpython-312.pyc,, +gitdb/test/__pycache__/test_pack.cpython-312.pyc,, +gitdb/test/__pycache__/test_stream.cpython-312.pyc,, +gitdb/test/__pycache__/test_util.cpython-312.pyc,, +gitdb/test/lib.py,sha256=iXBcJ3RDAN0uj9NbyEzt90TuCsDFt3SjHA61Ly6Xyxs,5489 +gitdb/test/test_base.py,sha256=6zjmON19J7LRDidlMSz2iQArONdkK6_v-DzhAQJzChg,2836 +gitdb/test/test_example.py,sha256=In7WgwvrGCbxPcuLCmkVIs4kdhCJ8rtcEvp115fHeU4,1350 +gitdb/test/test_pack.py,sha256=bbkupZPBFlpjBcPZTZ0OmRrTA9I3-fVdfwLtNa14_3s,9228 +gitdb/test/test_stream.py,sha256=RwSB49q6JHM7EFW413CZQeGRfkK92aQ0IVfTocIa3bo,5727 +gitdb/test/test_util.py,sha256=WQKQBP2uPF4wwNJFlUd9-YE2Q3CmlgpoY4MRk5G0r54,3243 +gitdb/typ.py,sha256=dZlbzfy5RFNHiZHEOwVy-6T-aZ3xLv0mGaJVyxKBd0M,373 +gitdb/util.py,sha256=F3bE24b2QDihz4tqc3Km7wbC6N65EjwFoao-yzrRJSY,12302 +gitdb/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +gitdb/utils/__pycache__/__init__.cpython-312.pyc,, +gitdb/utils/__pycache__/encoding.cpython-312.pyc,, +gitdb/utils/encoding.py,sha256=ceZZFb86LGJ71cwW6qkq_BFquAlNE7jaafNbwxYRSXk,372 diff --git a/lib/python3.12/site-packages/gitdb-4.0.12.dist-info/WHEEL b/lib/python3.12/site-packages/gitdb-4.0.12.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..ae527e7d64811439e61b93aa375defb30e06edfe --- /dev/null +++ b/lib/python3.12/site-packages/gitdb-4.0.12.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: setuptools (75.6.0) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/lib/python3.12/site-packages/gitdb-4.0.12.dist-info/top_level.txt b/lib/python3.12/site-packages/gitdb-4.0.12.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..c2023eef06a4fb9bb4a6e90d6405308215059169 --- /dev/null +++ b/lib/python3.12/site-packages/gitdb-4.0.12.dist-info/top_level.txt @@ -0,0 +1 @@ +gitdb diff --git a/lib/python3.12/site-packages/importlib_metadata-9.0.0.dist-info/INSTALLER b/lib/python3.12/site-packages/importlib_metadata-9.0.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.12/site-packages/importlib_metadata-9.0.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.12/site-packages/importlib_metadata-9.0.0.dist-info/METADATA b/lib/python3.12/site-packages/importlib_metadata-9.0.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..a9e787fed6a17c210987de0aae4e111ee640155c --- /dev/null +++ b/lib/python3.12/site-packages/importlib_metadata-9.0.0.dist-info/METADATA @@ -0,0 +1,130 @@ +Metadata-Version: 2.4 +Name: importlib_metadata +Version: 9.0.0 +Summary: Read metadata from Python packages +Author-email: "Jason R. Coombs" +License-Expression: Apache-2.0 +Project-URL: Source, https://github.com/python/importlib_metadata +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Requires-Python: >=3.10 +Description-Content-Type: text/x-rst +License-File: LICENSE +Requires-Dist: zipp>=3.20 +Provides-Extra: test +Requires-Dist: pytest!=8.1.*,>=6; extra == "test" +Requires-Dist: packaging; extra == "test" +Requires-Dist: pyfakefs; extra == "test" +Requires-Dist: pytest-perf>=0.9.2; extra == "test" +Provides-Extra: doc +Requires-Dist: sphinx>=3.5; extra == "doc" +Requires-Dist: jaraco.packaging>=9.3; extra == "doc" +Requires-Dist: rst.linker>=1.9; extra == "doc" +Requires-Dist: furo; extra == "doc" +Requires-Dist: sphinx-lint; extra == "doc" +Requires-Dist: jaraco.tidelift>=1.4; extra == "doc" +Provides-Extra: perf +Requires-Dist: ipython; extra == "perf" +Provides-Extra: check +Requires-Dist: pytest-checkdocs>=2.14; extra == "check" +Requires-Dist: pytest-ruff>=0.2.1; sys_platform != "cygwin" and extra == "check" +Provides-Extra: cover +Requires-Dist: pytest-cov; extra == "cover" +Provides-Extra: enabler +Requires-Dist: pytest-enabler>=3.4; extra == "enabler" +Provides-Extra: type +Requires-Dist: pytest-mypy>=1.0.1; platform_python_implementation != "PyPy" and extra == "type" +Dynamic: license-file + +.. image:: https://img.shields.io/pypi/v/importlib_metadata.svg + :target: https://pypi.org/project/importlib_metadata + +.. image:: https://img.shields.io/pypi/pyversions/importlib_metadata.svg + +.. image:: https://github.com/python/importlib_metadata/actions/workflows/main.yml/badge.svg + :target: https://github.com/python/importlib_metadata/actions?query=workflow%3A%22tests%22 + :alt: tests + +.. image:: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json + :target: https://github.com/astral-sh/ruff + :alt: Ruff + +.. image:: https://readthedocs.org/projects/importlib-metadata/badge/?version=latest + :target: https://importlib-metadata.readthedocs.io/en/latest/?badge=latest + +.. image:: https://img.shields.io/badge/skeleton-2025-informational + :target: https://blog.jaraco.com/skeleton + +.. image:: https://tidelift.com/badges/package/pypi/importlib-metadata + :target: https://tidelift.com/subscription/pkg/pypi-importlib-metadata?utm_source=pypi-importlib-metadata&utm_medium=readme + +Library to access the metadata for a Python package. + +This package supplies third-party access to the functionality of +`importlib.metadata `_ +including improvements added to subsequent Python versions. + + +Compatibility +============= + +New features are introduced in this third-party library and later merged +into CPython. The following table indicates which versions of this library +were contributed to different versions in the standard library: + +.. list-table:: + :header-rows: 1 + + * - importlib_metadata + - stdlib + * - 7.0 + - 3.13 + * - 6.5 + - 3.12 + * - 4.13 + - 3.11 + * - 4.6 + - 3.10 + * - 1.4 + - 3.8 + + +Usage +===== + +See the `online documentation `_ +for usage details. + +`Finder authors +`_ can +also add support for custom package installers. See the above documentation +for details. + + +Caveats +======= + +This project primarily supports third-party packages installed by PyPA +tools (or other conforming packages). It does not support: + +- Packages in the stdlib. +- Packages installed without metadata. + +Project details +=============== + + * Project home: https://github.com/python/importlib_metadata + * Report bugs at: https://github.com/python/importlib_metadata/issues + * Code hosting: https://github.com/python/importlib_metadata + * Documentation: https://importlib-metadata.readthedocs.io/ + +For Enterprise +============== + +Available as part of the Tidelift Subscription. + +This project and the maintainers of thousands of other packages are working with Tidelift to deliver one enterprise subscription that covers all of the open source you use. + +`Learn more `_. diff --git a/lib/python3.12/site-packages/importlib_metadata-9.0.0.dist-info/RECORD b/lib/python3.12/site-packages/importlib_metadata-9.0.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..44037f6dddf926c0257ac319133711fa3fb91b87 --- /dev/null +++ b/lib/python3.12/site-packages/importlib_metadata-9.0.0.dist-info/RECORD @@ -0,0 +1,31 @@ +importlib_metadata-9.0.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +importlib_metadata-9.0.0.dist-info/METADATA,sha256=xJvDjowQDA6j0KhllHqALIus5ES8DQ84YUGqadBCnH8,4536 +importlib_metadata-9.0.0.dist-info/RECORD,, +importlib_metadata-9.0.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91 +importlib_metadata-9.0.0.dist-info/licenses/LICENSE,sha256=4ve7BVfno4FJnxW6p8HXdtg9ZhZZkXnXHTEJuOOfeyU,10278 +importlib_metadata-9.0.0.dist-info/top_level.txt,sha256=CO3fD9yylANiXkrMo4qHLV_mqXL2sC5JFKgt1yWAT-A,19 +importlib_metadata/__init__.py,sha256=OiTiFNNx-2YUi3BHQMZR1gQncgzULAsEnykSajpwHrA,38281 +importlib_metadata/__pycache__/__init__.cpython-312.pyc,, +importlib_metadata/__pycache__/_adapters.cpython-312.pyc,, +importlib_metadata/__pycache__/_collections.cpython-312.pyc,, +importlib_metadata/__pycache__/_compat.cpython-312.pyc,, +importlib_metadata/__pycache__/_context.cpython-312.pyc,, +importlib_metadata/__pycache__/_functools.cpython-312.pyc,, +importlib_metadata/__pycache__/_itertools.cpython-312.pyc,, +importlib_metadata/__pycache__/_meta.cpython-312.pyc,, +importlib_metadata/__pycache__/_text.cpython-312.pyc,, +importlib_metadata/__pycache__/diagnose.cpython-312.pyc,, +importlib_metadata/_adapters.py,sha256=r5i8XLrKT6xmrpoREZhZrfczOYDmrVZeJBW5u0HzIGU,3797 +importlib_metadata/_collections.py,sha256=CxAhzlF3g1rwu_fMiB53JtRQiUFh0RgiMpoOvmK_ocg,760 +importlib_metadata/_compat.py,sha256=VC5ZDLlT-BcshauCShdFJvMNLntJJfZzNK1meGa-enw,1313 +importlib_metadata/_context.py,sha256=-QwKJg9f8ZqQhAnI4fb5kEcNiSQ0ZlclczLmCOWe_Sk,2672 +importlib_metadata/_functools.py,sha256=hVPD-xQZF68JXAhQOzDqpsD-gav28Ga0p1T_wDzLuwc,3537 +importlib_metadata/_itertools.py,sha256=nMvp9SfHAQ_JYwK4L2i64lr3GRXGlYlikGTVzWbys_E,5351 +importlib_metadata/_meta.py,sha256=EtHyiJ5kGzWFDfKyQ2XQp6Vu113CeadKW1Vf6aGc1B4,1765 +importlib_metadata/_text.py,sha256=HCsFksZpJLeTP3NEk_ngrAeXVRRtTrtyh9eOABoRP4A,2166 +importlib_metadata/compat/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +importlib_metadata/compat/__pycache__/__init__.cpython-312.pyc,, +importlib_metadata/compat/__pycache__/py311.cpython-312.pyc,, +importlib_metadata/compat/py311.py,sha256=uqm-K-uohyj1042TH4a9Er_I5o7667DvulcD-gC_fSA,608 +importlib_metadata/diagnose.py,sha256=nkSRMiowlmkhLYhKhvCg9glmt_11Cox-EmLzEbqYTa8,379 +importlib_metadata/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 diff --git a/lib/python3.12/site-packages/importlib_metadata-9.0.0.dist-info/WHEEL b/lib/python3.12/site-packages/importlib_metadata-9.0.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..14a883f292bc96b20c2b76a3081991f2676523a9 --- /dev/null +++ b/lib/python3.12/site-packages/importlib_metadata-9.0.0.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: setuptools (82.0.1) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/lib/python3.12/site-packages/importlib_metadata-9.0.0.dist-info/licenses/LICENSE b/lib/python3.12/site-packages/importlib_metadata-9.0.0.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..7b25cfb85f10bb21140ba18e5df5087d75406dd5 --- /dev/null +++ b/lib/python3.12/site-packages/importlib_metadata-9.0.0.dist-info/licenses/LICENSE @@ -0,0 +1,73 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + + (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + + You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. + +Copyright 2026 [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/lib/python3.12/site-packages/importlib_metadata-9.0.0.dist-info/top_level.txt b/lib/python3.12/site-packages/importlib_metadata-9.0.0.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..bbb07547a19c30031d13c45cf01cba61dc434e47 --- /dev/null +++ b/lib/python3.12/site-packages/importlib_metadata-9.0.0.dist-info/top_level.txt @@ -0,0 +1 @@ +importlib_metadata diff --git a/lib/python3.12/site-packages/jedi/__init__.py b/lib/python3.12/site-packages/jedi/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..70b184ac0e3c587993eed620d03c60a951a5c761 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/__init__.py @@ -0,0 +1,42 @@ +""" +Jedi is a static analysis tool for Python that is typically used in +IDEs/editors plugins. Jedi has a focus on autocompletion and goto +functionality. Other features include refactoring, code search and finding +references. + +Jedi has a simple API to work with. There is a reference implementation as a +`VIM-Plugin `_. Autocompletion in your +REPL is also possible, IPython uses it natively and for the CPython REPL you +can install it. Jedi is well tested and bugs should be rare. + +Here's a simple example of the autocompletion feature: + +>>> import jedi +>>> source = ''' +... import json +... json.lo''' +>>> script = jedi.Script(source, path='example.py') +>>> script + +>>> completions = script.complete(3, len('json.lo')) +>>> completions +[, ] +>>> print(completions[0].complete) +ad +>>> print(completions[0].name) +load +""" + +__version__ = '0.19.2' + +from jedi.api import Script, Interpreter, set_debug_function, preload_module +from jedi import settings +from jedi.api.environment import find_virtualenvs, find_system_environments, \ + get_default_environment, InvalidPythonEnvironment, create_environment, \ + get_system_environment, InterpreterEnvironment +from jedi.api.project import Project, get_default_project +from jedi.api.exceptions import InternalError, RefactoringError + +# Finally load the internal plugins. This is only internal. +from jedi.plugins import registry +del registry diff --git a/lib/python3.12/site-packages/jedi/__main__.py b/lib/python3.12/site-packages/jedi/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..6b4425333e2a33c3a89318da6b25dd7c4cc666d5 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/__main__.py @@ -0,0 +1,72 @@ +import sys +from os.path import join, dirname, abspath, isdir + + +def _start_linter(): + """ + This is a pre-alpha API. You're not supposed to use it at all, except for + testing. It will very likely change. + """ + import jedi + + if '--debug' in sys.argv: + jedi.set_debug_function() + + for path in sys.argv[2:]: + if path.startswith('--'): + continue + if isdir(path): + import fnmatch + import os + + paths = [] + for root, dirnames, filenames in os.walk(path): + for filename in fnmatch.filter(filenames, '*.py'): + paths.append(os.path.join(root, filename)) + else: + paths = [path] + + try: + for p in paths: + for error in jedi.Script(path=p)._analysis(): + print(error) + except Exception: + if '--pdb' in sys.argv: + import traceback + traceback.print_exc() + import pdb + pdb.post_mortem() + else: + raise + + +def _complete(): + import jedi + import pdb + + if '-d' in sys.argv: + sys.argv.remove('-d') + jedi.set_debug_function() + + try: + completions = jedi.Script(sys.argv[2]).complete() + for c in completions: + c.docstring() + c.type + except Exception as e: + print(repr(e)) + pdb.post_mortem() + else: + print(completions) + + +if len(sys.argv) == 2 and sys.argv[1] == 'repl': + # don't want to use __main__ only for repl yet, maybe we want to use it for + # something else. So just use the keyword ``repl`` for now. + print(join(dirname(abspath(__file__)), 'api', 'replstartup.py')) +elif len(sys.argv) > 1 and sys.argv[1] == '_linter': + _start_linter() +elif len(sys.argv) > 1 and sys.argv[1] == '_complete': + _complete() +else: + print('Command not implemented: %s' % sys.argv[1]) diff --git a/lib/python3.12/site-packages/jedi/_compatibility.py b/lib/python3.12/site-packages/jedi/_compatibility.py new file mode 100644 index 0000000000000000000000000000000000000000..48563d00ddd11b99961164df5924d5e19518f8c3 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/_compatibility.py @@ -0,0 +1,45 @@ +""" +This module is here to ensure compatibility of Windows/Linux/MacOS and +different Python versions. +""" +import errno +import sys +import pickle +from typing import Any + + +class Unpickler(pickle.Unpickler): + def find_class(self, module: str, name: str) -> Any: + # Python 3.13 moved pathlib implementation out of __init__.py as part of + # generalising its implementation. Ensure that we support loading + # pickles from 3.13 on older version of Python. Since 3.13 maintained a + # compatible API, pickles from older Python work natively on the newer + # version. + if module == 'pathlib._local': + module = 'pathlib' + return super().find_class(module, name) + + +def pickle_load(file): + try: + return Unpickler(file).load() + # Python on Windows don't throw EOF errors for pipes. So reraise them with + # the correct type, which is caught upwards. + except OSError: + if sys.platform == 'win32': + raise EOFError() + raise + + +def pickle_dump(data, file, protocol): + try: + pickle.dump(data, file, protocol) + # On Python 3.3 flush throws sometimes an error even though the writing + # operation should be completed. + file.flush() + # Python on Windows don't throw EPIPE errors for pipes. So reraise them with + # the correct type and error number. + except OSError: + if sys.platform == 'win32': + raise IOError(errno.EPIPE, "Broken pipe") + raise diff --git a/lib/python3.12/site-packages/jedi/cache.py b/lib/python3.12/site-packages/jedi/cache.py new file mode 100644 index 0000000000000000000000000000000000000000..1ff452013f8bfa90753296dea65a79f4e56a2689 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/cache.py @@ -0,0 +1,115 @@ +""" +This caching is very important for speed and memory optimizations. There's +nothing really spectacular, just some decorators. The following cache types are +available: + +- ``time_cache`` can be used to cache something for just a limited time span, + which can be useful if there's user interaction and the user cannot react + faster than a certain time. + +This module is one of the reasons why |jedi| is not thread-safe. As you can see +there are global variables, which are holding the cache information. Some of +these variables are being cleaned after every API usage. +""" +import time +from functools import wraps +from typing import Any, Dict, Tuple + +from jedi import settings +from parso.cache import parser_cache + +_time_caches: Dict[str, Dict[Any, Tuple[float, Any]]] = {} + + +def clear_time_caches(delete_all: bool = False) -> None: + """ Jedi caches many things, that should be completed after each completion + finishes. + + :param delete_all: Deletes also the cache that is normally not deleted, + like parser cache, which is important for faster parsing. + """ + global _time_caches + + if delete_all: + for cache in _time_caches.values(): + cache.clear() + parser_cache.clear() + else: + # normally just kill the expired entries, not all + for tc in _time_caches.values(): + # check time_cache for expired entries + for key, (t, value) in list(tc.items()): + if t < time.time(): + # delete expired entries + del tc[key] + + +def signature_time_cache(time_add_setting): + """ + This decorator works as follows: Call it with a setting and after that + use the function with a callable that returns the key. + But: This function is only called if the key is not available. After a + certain amount of time (`time_add_setting`) the cache is invalid. + + If the given key is None, the function will not be cached. + """ + def _temp(key_func): + dct = {} + _time_caches[time_add_setting] = dct + + def wrapper(*args, **kwargs): + generator = key_func(*args, **kwargs) + key = next(generator) + try: + expiry, value = dct[key] + if expiry > time.time(): + return value + except KeyError: + pass + + value = next(generator) + time_add = getattr(settings, time_add_setting) + if key is not None: + dct[key] = time.time() + time_add, value + return value + return wrapper + return _temp + + +def time_cache(seconds): + def decorator(func): + cache = {} + + @wraps(func) + def wrapper(*args, **kwargs): + key = (args, frozenset(kwargs.items())) + try: + created, result = cache[key] + if time.time() < created + seconds: + return result + except KeyError: + pass + result = func(*args, **kwargs) + cache[key] = time.time(), result + return result + + wrapper.clear_cache = lambda: cache.clear() + return wrapper + + return decorator + + +def memoize_method(method): + """A normal memoize function.""" + @wraps(method) + def wrapper(self, *args, **kwargs): + cache_dict = self.__dict__.setdefault('_memoize_method_dct', {}) + dct = cache_dict.setdefault(method, {}) + key = (args, frozenset(kwargs.items())) + try: + return dct[key] + except KeyError: + result = method(self, *args, **kwargs) + dct[key] = result + return result + return wrapper diff --git a/lib/python3.12/site-packages/jedi/common.py b/lib/python3.12/site-packages/jedi/common.py new file mode 100644 index 0000000000000000000000000000000000000000..eb4b49961c8e3f189b42bec0aba10a559f7ad4cf --- /dev/null +++ b/lib/python3.12/site-packages/jedi/common.py @@ -0,0 +1,24 @@ +from contextlib import contextmanager + + +@contextmanager +def monkeypatch(obj, attribute_name, new_value): + """ + Like pytest's monkeypatch, but as a value manager. + """ + old_value = getattr(obj, attribute_name) + try: + setattr(obj, attribute_name, new_value) + yield + finally: + setattr(obj, attribute_name, old_value) + + +def indent_block(text, indention=' '): + """This function indents a text block with a default of four spaces.""" + temp = '' + while text and text[-1] == '\n': + temp += text[-1] + text = text[:-1] + lines = text.split('\n') + return '\n'.join(map(lambda s: indention + s, lines)) + temp diff --git a/lib/python3.12/site-packages/jedi/debug.py b/lib/python3.12/site-packages/jedi/debug.py new file mode 100644 index 0000000000000000000000000000000000000000..99a90601e847bac76f9af7bdee020f5f67bc611b --- /dev/null +++ b/lib/python3.12/site-packages/jedi/debug.py @@ -0,0 +1,132 @@ +import os +import time +from contextlib import contextmanager +from typing import Callable, Optional + +_inited = False + + +def _lazy_colorama_init(): + """ + Lazily init colorama if necessary, not to screw up stdout if debugging is + not enabled. + + This version of the function does nothing. + """ + + +try: + if os.name == 'nt': + # Does not work on Windows, as pyreadline and colorama interfere + raise ImportError + else: + # Use colorama for nicer console output. + from colorama import Fore, init # type: ignore[import] + from colorama import initialise + + def _lazy_colorama_init(): # noqa: F811 + """ + Lazily init colorama if necessary, not to screw up stdout is + debug not enabled. + + This version of the function does init colorama. + """ + global _inited + if not _inited: + # pytest resets the stream at the end - causes troubles. Since + # after every output the stream is reset automatically we don't + # need this. + initialise.atexit_done = True + try: + init(strip=False) + except Exception: + # Colorama fails with initializing under vim and is buggy in + # version 0.3.6. + pass + _inited = True + +except ImportError: + class Fore: # type: ignore[no-redef] + RED = '' + GREEN = '' + YELLOW = '' + MAGENTA = '' + RESET = '' + BLUE = '' + +NOTICE = object() +WARNING = object() +SPEED = object() + +enable_speed = False +enable_warning = False +enable_notice = False + +# callback, interface: level, str +debug_function: Optional[Callable[[str, str], None]] = None +_debug_indent = 0 +_start_time = time.time() + + +def reset_time(): + global _start_time, _debug_indent + _start_time = time.time() + _debug_indent = 0 + + +def increase_indent(func): + """Decorator for makin """ + def wrapper(*args, **kwargs): + with increase_indent_cm(): + return func(*args, **kwargs) + return wrapper + + +@contextmanager +def increase_indent_cm(title=None, color='MAGENTA'): + global _debug_indent + if title: + dbg('Start: ' + title, color=color) + _debug_indent += 1 + try: + yield + finally: + _debug_indent -= 1 + if title: + dbg('End: ' + title, color=color) + + +def dbg(message, *args, color='GREEN'): + """ Looks at the stack, to see if a debug message should be printed. """ + assert color + + if debug_function and enable_notice: + i = ' ' * _debug_indent + _lazy_colorama_init() + debug_function(color, i + 'dbg: ' + message % tuple(repr(a) for a in args)) + + +def warning(message, *args, format=True): + if debug_function and enable_warning: + i = ' ' * _debug_indent + if format: + message = message % tuple(repr(a) for a in args) + debug_function('RED', i + 'warning: ' + message) + + +def speed(name): + if debug_function and enable_speed: + now = time.time() + i = ' ' * _debug_indent + debug_function('YELLOW', i + 'speed: ' + '%s %s' % (name, now - _start_time)) + + +def print_to_stdout(color, str_out): + """ + The default debug function that prints to standard out. + + :param str color: A string that is an attribute of ``colorama.Fore``. + """ + col = getattr(Fore, color) + _lazy_colorama_init() + print(col + str_out + Fore.RESET) diff --git a/lib/python3.12/site-packages/jedi/file_io.py b/lib/python3.12/site-packages/jedi/file_io.py new file mode 100644 index 0000000000000000000000000000000000000000..ead17335cfd3011ad3a4a3100aee63002a559901 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/file_io.py @@ -0,0 +1,83 @@ +import os + +from parso import file_io + + +class AbstractFolderIO: + def __init__(self, path): + self.path = path + + def get_base_name(self): + raise NotImplementedError + + def list(self): + raise NotImplementedError + + def get_file_io(self, name): + raise NotImplementedError + + def get_parent_folder(self): + raise NotImplementedError + + def __repr__(self): + return '<%s: %s>' % (self.__class__.__name__, self.path) + + +class FolderIO(AbstractFolderIO): + def get_base_name(self): + return os.path.basename(self.path) + + def list(self): + return os.listdir(self.path) + + def get_file_io(self, name): + return FileIO(os.path.join(self.path, name)) + + def get_parent_folder(self): + return FolderIO(os.path.dirname(self.path)) + + def walk(self): + for root, dirs, files in os.walk(self.path): + root_folder_io = FolderIO(root) + original_folder_ios = [FolderIO(os.path.join(root, d)) for d in dirs] + modified_folder_ios = list(original_folder_ios) + yield ( + root_folder_io, + modified_folder_ios, + [FileIO(os.path.join(root, f)) for f in files], + ) + modified_iterator = iter(reversed(modified_folder_ios)) + current = next(modified_iterator, None) + i = len(original_folder_ios) + for folder_io in reversed(original_folder_ios): + i -= 1 # Basically enumerate but reversed + if current is folder_io: + current = next(modified_iterator, None) + else: + del dirs[i] + + +class FileIOFolderMixin: + def get_parent_folder(self): + return FolderIO(os.path.dirname(self.path)) + + +class ZipFileIO(file_io.KnownContentFileIO, FileIOFolderMixin): + """For .zip and .egg archives""" + def __init__(self, path, code, zip_path): + super().__init__(path, code) + self._zip_path = zip_path + + def get_last_modified(self): + try: + return os.path.getmtime(self._zip_path) + except (FileNotFoundError, PermissionError, NotADirectoryError): + return None + + +class FileIO(file_io.FileIO, FileIOFolderMixin): + pass + + +class KnownContentFileIO(file_io.KnownContentFileIO, FileIOFolderMixin): + pass diff --git a/lib/python3.12/site-packages/jedi/parser_utils.py b/lib/python3.12/site-packages/jedi/parser_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ddec28e8c4956722207cb28bda0478f3fa5c680e --- /dev/null +++ b/lib/python3.12/site-packages/jedi/parser_utils.py @@ -0,0 +1,345 @@ +import re +import textwrap +from ast import literal_eval +from inspect import cleandoc +from weakref import WeakKeyDictionary + +from parso.python import tree +from parso.cache import parser_cache +from parso import split_lines + +_EXECUTE_NODES = {'funcdef', 'classdef', 'import_from', 'import_name', 'test', + 'or_test', 'and_test', 'not_test', 'comparison', 'expr', + 'xor_expr', 'and_expr', 'shift_expr', 'arith_expr', + 'atom_expr', 'term', 'factor', 'power', 'atom'} + +_FLOW_KEYWORDS = ( + 'try', 'except', 'finally', 'else', 'if', 'elif', 'with', 'for', 'while' +) + + +def get_executable_nodes(node, last_added=False): + """ + For static analysis. + """ + result = [] + typ = node.type + if typ == 'name': + next_leaf = node.get_next_leaf() + if last_added is False and node.parent.type != 'param' and next_leaf != '=': + result.append(node) + elif typ == 'expr_stmt': + # I think inferring the statement (and possibly returned arrays), + # should be enough for static analysis. + result.append(node) + for child in node.children: + result += get_executable_nodes(child, last_added=True) + elif typ == 'decorator': + # decorator + if node.children[-2] == ')': + node = node.children[-3] + if node != '(': + result += get_executable_nodes(node) + else: + try: + children = node.children + except AttributeError: + pass + else: + if node.type in _EXECUTE_NODES and not last_added: + result.append(node) + + for child in children: + result += get_executable_nodes(child, last_added) + + return result + + +def get_sync_comp_fors(comp_for): + yield comp_for + last = comp_for.children[-1] + while True: + if last.type == 'comp_for': + yield last.children[1] # Ignore the async. + elif last.type == 'sync_comp_for': + yield last + elif not last.type == 'comp_if': + break + last = last.children[-1] + + +def for_stmt_defines_one_name(for_stmt): + """ + Returns True if only one name is returned: ``for x in y``. + Returns False if the for loop is more complicated: ``for x, z in y``. + + :returns: bool + """ + return for_stmt.children[1].type == 'name' + + +def get_flow_branch_keyword(flow_node, node): + start_pos = node.start_pos + if not (flow_node.start_pos < start_pos <= flow_node.end_pos): + raise ValueError('The node is not part of the flow.') + + keyword = None + for i, child in enumerate(flow_node.children): + if start_pos < child.start_pos: + return keyword + first_leaf = child.get_first_leaf() + if first_leaf in _FLOW_KEYWORDS: + keyword = first_leaf + return None + + +def clean_scope_docstring(scope_node): + """ Returns a cleaned version of the docstring token. """ + node = scope_node.get_doc_node() + if node is not None: + # TODO We have to check next leaves until there are no new + # leaves anymore that might be part of the docstring. A + # docstring can also look like this: ``'foo' 'bar' + # Returns a literal cleaned version of the ``Token``. + return cleandoc(safe_literal_eval(node.value)) + return '' + + +def find_statement_documentation(tree_node): + if tree_node.type == 'expr_stmt': + tree_node = tree_node.parent # simple_stmt + maybe_string = tree_node.get_next_sibling() + if maybe_string is not None: + if maybe_string.type == 'simple_stmt': + maybe_string = maybe_string.children[0] + if maybe_string.type == 'string': + return cleandoc(safe_literal_eval(maybe_string.value)) + return '' + + +def safe_literal_eval(value): + first_two = value[:2].lower() + if first_two[0] == 'f' or first_two in ('fr', 'rf'): + # literal_eval is not able to resovle f literals. We have to do that + # manually, but that's right now not implemented. + return '' + + return literal_eval(value) + + +def get_signature(funcdef, width=72, call_string=None, + omit_first_param=False, omit_return_annotation=False): + """ + Generate a string signature of a function. + + :param width: Fold lines if a line is longer than this value. + :type width: int + :arg func_name: Override function name when given. + :type func_name: str + + :rtype: str + """ + # Lambdas have no name. + if call_string is None: + if funcdef.type == 'lambdef': + call_string = '' + else: + call_string = funcdef.name.value + params = funcdef.get_params() + if omit_first_param: + params = params[1:] + p = '(' + ''.join(param.get_code() for param in params).strip() + ')' + # TODO this is pretty bad, we should probably just normalize. + p = re.sub(r'\s+', ' ', p) + if funcdef.annotation and not omit_return_annotation: + rtype = " ->" + funcdef.annotation.get_code() + else: + rtype = "" + code = call_string + p + rtype + + return '\n'.join(textwrap.wrap(code, width)) + + +def move(node, line_offset): + """ + Move the `Node` start_pos. + """ + try: + children = node.children + except AttributeError: + node.line += line_offset + else: + for c in children: + move(c, line_offset) + + +def get_following_comment_same_line(node): + """ + returns (as string) any comment that appears on the same line, + after the node, including the # + """ + try: + if node.type == 'for_stmt': + whitespace = node.children[5].get_first_leaf().prefix + elif node.type == 'with_stmt': + whitespace = node.children[3].get_first_leaf().prefix + elif node.type == 'funcdef': + # actually on the next line + whitespace = node.children[4].get_first_leaf().get_next_leaf().prefix + else: + whitespace = node.get_last_leaf().get_next_leaf().prefix + except AttributeError: + return None + except ValueError: + # TODO in some particular cases, the tree doesn't seem to be linked + # correctly + return None + if "#" not in whitespace: + return None + comment = whitespace[whitespace.index("#"):] + if "\r" in comment: + comment = comment[:comment.index("\r")] + if "\n" in comment: + comment = comment[:comment.index("\n")] + return comment + + +def is_scope(node): + t = node.type + if t == 'comp_for': + # Starting with Python 3.8, async is outside of the statement. + return node.children[1].type != 'sync_comp_for' + + return t in ('file_input', 'classdef', 'funcdef', 'lambdef', 'sync_comp_for') + + +def _get_parent_scope_cache(func): + cache = WeakKeyDictionary() + + def wrapper(parso_cache_node, node, include_flows=False): + if parso_cache_node is None: + return func(node, include_flows) + + try: + for_module = cache[parso_cache_node] + except KeyError: + for_module = cache[parso_cache_node] = {} + + try: + return for_module[node] + except KeyError: + result = for_module[node] = func(node, include_flows) + return result + return wrapper + + +def get_parent_scope(node, include_flows=False): + """ + Returns the underlying scope. + """ + scope = node.parent + if scope is None: + return None # It's a module already. + + while True: + if is_scope(scope): + if scope.type in ('classdef', 'funcdef', 'lambdef'): + index = scope.children.index(':') + if scope.children[index].start_pos >= node.start_pos: + if node.parent.type == 'param' and node.parent.name == node: + pass + elif node.parent.type == 'tfpdef' and node.parent.children[0] == node: + pass + else: + scope = scope.parent + continue + return scope + elif include_flows and isinstance(scope, tree.Flow): + # The cursor might be on `if foo`, so the parent scope will not be + # the if, but the parent of the if. + if not (scope.type == 'if_stmt' + and any(n.start_pos <= node.start_pos < n.end_pos + for n in scope.get_test_nodes())): + return scope + + scope = scope.parent + + +get_cached_parent_scope = _get_parent_scope_cache(get_parent_scope) + + +def get_cached_code_lines(grammar, path): + """ + Basically access the cached code lines in parso. This is not the nicest way + to do this, but we avoid splitting all the lines again. + """ + return get_parso_cache_node(grammar, path).lines + + +def get_parso_cache_node(grammar, path): + """ + This is of course not public. But as long as I control parso, this + shouldn't be a problem. ~ Dave + + The reason for this is mostly caching. This is obviously also a sign of a + broken caching architecture. + """ + return parser_cache[grammar._hashed][path] + + +def cut_value_at_position(leaf, position): + """ + Cuts of the value of the leaf at position + """ + lines = split_lines(leaf.value, keepends=True)[:position[0] - leaf.line + 1] + column = position[1] + if leaf.line == position[0]: + column -= leaf.column + if not lines: + return '' + lines[-1] = lines[-1][:column] + return ''.join(lines) + + +def expr_is_dotted(node): + """ + Checks if a path looks like `name` or `name.foo.bar` and not `name()`. + """ + if node.type == 'atom': + if len(node.children) == 3 and node.children[0] == '(': + return expr_is_dotted(node.children[1]) + return False + if node.type == 'atom_expr': + children = node.children + if children[0] == 'await': + return False + if not expr_is_dotted(children[0]): + return False + # Check trailers + return all(c.children[0] == '.' for c in children[1:]) + return node.type == 'name' + + +def _function_is_x_method(decorator_checker): + def wrapper(function_node): + """ + This is a heuristic. It will not hold ALL the times, but it will be + correct pretty much for anyone that doesn't try to beat it. + staticmethod/classmethod are builtins and unless overwritten, this will + be correct. + """ + for decorator in function_node.get_decorators(): + dotted_name = decorator.children[1] + if decorator_checker(dotted_name.get_code()): + return True + return False + return wrapper + + +function_is_staticmethod = _function_is_x_method(lambda m: m == "staticmethod") +function_is_classmethod = _function_is_x_method(lambda m: m == "classmethod") +function_is_property = _function_is_x_method( + lambda m: m == "property" + or m == "cached_property" + or (m.endswith(".setter")) +) diff --git a/lib/python3.12/site-packages/jedi/settings.py b/lib/python3.12/site-packages/jedi/settings.py new file mode 100644 index 0000000000000000000000000000000000000000..3a333e01a110bd090027ac48274b94cdd525460e --- /dev/null +++ b/lib/python3.12/site-packages/jedi/settings.py @@ -0,0 +1,163 @@ +""" +This module contains variables with global |jedi| settings. To change the +behavior of |jedi|, change the variables defined in :mod:`jedi.settings`. + +Plugins should expose an interface so that the user can adjust the +configuration. + + +Example usage:: + + from jedi import settings + settings.case_insensitive_completion = True + + +Completion output +~~~~~~~~~~~~~~~~~ + +.. autodata:: case_insensitive_completion +.. autodata:: add_bracket_after_function + + +Filesystem cache +~~~~~~~~~~~~~~~~ + +.. autodata:: cache_directory + + +Parser +~~~~~~ + +.. autodata:: fast_parser + + +Dynamic stuff +~~~~~~~~~~~~~ + +.. autodata:: dynamic_array_additions +.. autodata:: dynamic_params +.. autodata:: dynamic_params_for_other_modules +.. autodata:: auto_import_modules + + +Caching +~~~~~~~ + +.. autodata:: call_signatures_validity + + +""" +import os +import platform + +# ---------------- +# Completion Output Settings +# ---------------- + +case_insensitive_completion = True +""" +Completions are by default case insensitive. +""" + +add_bracket_after_function = False +""" +Adds an opening bracket after a function for completions. +""" + +# ---------------- +# Filesystem Cache +# ---------------- + +if platform.system().lower() == 'windows': + _cache_directory = os.path.join( + os.getenv('LOCALAPPDATA') or os.path.expanduser('~'), + 'Jedi', + 'Jedi', + ) +elif platform.system().lower() == 'darwin': + _cache_directory = os.path.join('~', 'Library', 'Caches', 'Jedi') +else: + _cache_directory = os.path.join(os.getenv('XDG_CACHE_HOME') or '~/.cache', + 'jedi') +cache_directory = os.path.expanduser(_cache_directory) +""" +The path where the cache is stored. + +On Linux, this defaults to ``~/.cache/jedi/``, on OS X to +``~/Library/Caches/Jedi/`` and on Windows to ``%LOCALAPPDATA%\\Jedi\\Jedi\\``. +On Linux, if the environment variable ``$XDG_CACHE_HOME`` is set, +``$XDG_CACHE_HOME/jedi`` is used instead of the default one. +""" + +# ---------------- +# Parser +# ---------------- + +fast_parser = True +""" +Uses Parso's diff parser. If it is enabled, this might cause issues, please +read the warning on :class:`.Script`. This feature makes it possible to only +parse the parts again that have changed, while reusing the rest of the syntax +tree. +""" + +_cropped_file_size = int(10e6) # 1 Megabyte +""" +Jedi gets extremely slow if the file size exceed a few thousand lines. +To avoid getting stuck completely Jedi crops the file at some point. + +One megabyte of typical Python code equals about 20'000 lines of code. +""" + +# ---------------- +# Dynamic Stuff +# ---------------- + +dynamic_array_additions = True +""" +check for `append`, etc. on arrays: [], {}, () as well as list/set calls. +""" + +dynamic_params = True +""" +A dynamic param completion, finds the callees of the function, which define +the params of a function. +""" + +dynamic_params_for_other_modules = True +""" +Do the same for other modules. +""" + +dynamic_flow_information = True +""" +Check for `isinstance` and other information to infer a type. +""" + +auto_import_modules = [ + 'gi', # This third-party repository (GTK stuff) doesn't really work with jedi +] +""" +Modules that will not be analyzed but imported, if they contain Python code. +This improves autocompletion for libraries that use ``setattr`` or +``globals()`` modifications a lot. +""" + +allow_unsafe_interpreter_executions = True +""" +Controls whether descriptors are evaluated when using an Interpreter. This is +something you might want to control when using Jedi from a Repl (e.g. IPython) + +Generally this setting allows Jedi to execute __getitem__ and descriptors like +`property`. +""" + +# ---------------- +# Caching Validity +# ---------------- + +call_signatures_validity = 3.0 +""" +Finding function calls might be slow (0.1-0.5s). This is not acceptible for +normal writing. Therefore cache it for a short time. +""" diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/LICENSE b/lib/python3.12/site-packages/jedi/third_party/typeshed/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..e5833ae4231d68a2cc6878b790199e1d97900b39 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/LICENSE @@ -0,0 +1,238 @@ +The "typeshed" project is licensed under the terms of the Apache license, as +reproduced below. + += = = = = + +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + += = = = = + +Parts of typeshed are licensed under different licenses (like the MIT +license), reproduced below. + += = = = = + +The MIT License + +Copyright (c) 2015 Jukka Lehtosalo and contributors + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + += = = = = + diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/contextvars.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/contextvars.pyi new file mode 100644 index 0000000000000000000000000000000000000000..429d2037afb2bbbc813696f1941f7fcdce6e34ca --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/contextvars.pyi @@ -0,0 +1,38 @@ +import sys +from typing import Any, Callable, ClassVar, Generic, Iterator, Mapping, TypeVar + +if sys.version_info >= (3, 9): + from types import GenericAlias + +_T = TypeVar("_T") + +class ContextVar(Generic[_T]): + def __init__(self, name: str, *, default: _T = ...) -> None: ... + @property + def name(self) -> str: ... + def get(self, default: _T = ...) -> _T: ... + def set(self, value: _T) -> Token[_T]: ... + def reset(self, token: Token[_T]) -> None: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any) -> GenericAlias: ... + +class Token(Generic[_T]): + @property + def var(self) -> ContextVar[_T]: ... + @property + def old_value(self) -> Any: ... # returns either _T or MISSING, but that's hard to express + MISSING: ClassVar[object] + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any) -> GenericAlias: ... + +def copy_context() -> Context: ... + +# It doesn't make sense to make this generic, because for most Contexts each ContextVar will have +# a different value. +class Context(Mapping[ContextVar[Any], Any]): + def __init__(self) -> None: ... + def run(self, callable: Callable[..., _T], *args: Any, **kwargs: Any) -> _T: ... + def copy(self) -> Context: ... + def __getitem__(self, key: ContextVar[Any]) -> Any: ... + def __iter__(self) -> Iterator[ContextVar[Any]]: ... + def __len__(self) -> int: ... diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/dataclasses.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/dataclasses.pyi new file mode 100644 index 0000000000000000000000000000000000000000..1f5b6b4d35b923e3c1a444573cb26623cc820eb0 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/dataclasses.pyi @@ -0,0 +1,95 @@ +import sys +from typing import Any, Callable, Dict, Generic, Iterable, List, Mapping, Optional, Tuple, Type, TypeVar, Union, overload + +if sys.version_info >= (3, 9): + from types import GenericAlias + +_T = TypeVar("_T") + +class _MISSING_TYPE: ... + +MISSING: _MISSING_TYPE +@overload +def asdict(obj: Any) -> Dict[str, Any]: ... +@overload +def asdict(obj: Any, *, dict_factory: Callable[[List[Tuple[str, Any]]], _T]) -> _T: ... +@overload +def astuple(obj: Any) -> Tuple[Any, ...]: ... +@overload +def astuple(obj: Any, *, tuple_factory: Callable[[List[Any]], _T]) -> _T: ... +@overload +def dataclass(_cls: Type[_T]) -> Type[_T]: ... +@overload +def dataclass(_cls: None) -> Callable[[Type[_T]], Type[_T]]: ... +@overload +def dataclass( + *, init: bool = ..., repr: bool = ..., eq: bool = ..., order: bool = ..., unsafe_hash: bool = ..., frozen: bool = ... +) -> Callable[[Type[_T]], Type[_T]]: ... + +class Field(Generic[_T]): + name: str + type: Type[_T] + default: _T + default_factory: Callable[[], _T] + repr: bool + hash: Optional[bool] + init: bool + compare: bool + metadata: Mapping[str, Any] + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any) -> GenericAlias: ... + +# NOTE: Actual return type is 'Field[_T]', but we want to help type checkers +# to understand the magic that happens at runtime. +@overload # `default` and `default_factory` are optional and mutually exclusive. +def field( + *, + default: _T, + init: bool = ..., + repr: bool = ..., + hash: Optional[bool] = ..., + compare: bool = ..., + metadata: Optional[Mapping[str, Any]] = ..., +) -> _T: ... +@overload +def field( + *, + default_factory: Callable[[], _T], + init: bool = ..., + repr: bool = ..., + hash: Optional[bool] = ..., + compare: bool = ..., + metadata: Optional[Mapping[str, Any]] = ..., +) -> _T: ... +@overload +def field( + *, + init: bool = ..., + repr: bool = ..., + hash: Optional[bool] = ..., + compare: bool = ..., + metadata: Optional[Mapping[str, Any]] = ..., +) -> Any: ... +def fields(class_or_instance: Any) -> Tuple[Field[Any], ...]: ... +def is_dataclass(obj: Any) -> bool: ... + +class FrozenInstanceError(AttributeError): ... + +class InitVar(Generic[_T]): + if sys.version_info >= (3, 9): + def __class_getitem__(cls, type: Any) -> GenericAlias: ... + +def make_dataclass( + cls_name: str, + fields: Iterable[Union[str, Tuple[str, type], Tuple[str, type, Field[Any]]]], + *, + bases: Tuple[type, ...] = ..., + namespace: Optional[Dict[str, Any]] = ..., + init: bool = ..., + repr: bool = ..., + eq: bool = ..., + order: bool = ..., + unsafe_hash: bool = ..., + frozen: bool = ..., +) -> type: ... +def replace(obj: _T, **changes: Any) -> _T: ... diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/docutils/__init__.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/docutils/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..e27843e5338213713e26973127c738c14313ff98 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/docutils/__init__.pyi @@ -0,0 +1,3 @@ +from typing import Any + +def __getattr__(name: str) -> Any: ... diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/docutils/examples.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/docutils/examples.pyi new file mode 100644 index 0000000000000000000000000000000000000000..581ebba7e61e4a3d2473342beab51f70c76384c7 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/docutils/examples.pyi @@ -0,0 +1,5 @@ +from typing import Any + +html_parts: Any + +def __getattr__(name: str) -> Any: ... diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/docutils/nodes.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/docutils/nodes.pyi new file mode 100644 index 0000000000000000000000000000000000000000..11773347ff8759548a696ad7a3d3d4972c9fcb5e --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/docutils/nodes.pyi @@ -0,0 +1,6 @@ +from typing import Any, List + +class reference: + def __init__(self, rawsource: str = ..., text: str = ..., *children: List[Any], **attributes: Any) -> None: ... + +def __getattr__(name: str) -> Any: ... diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/docutils/parsers/__init__.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/docutils/parsers/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..e27843e5338213713e26973127c738c14313ff98 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/docutils/parsers/__init__.pyi @@ -0,0 +1,3 @@ +from typing import Any + +def __getattr__(name: str) -> Any: ... diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/docutils/parsers/rst/__init__.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/docutils/parsers/rst/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..e27843e5338213713e26973127c738c14313ff98 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/docutils/parsers/rst/__init__.pyi @@ -0,0 +1,3 @@ +from typing import Any + +def __getattr__(name: str) -> Any: ... diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/docutils/parsers/rst/nodes.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/docutils/parsers/rst/nodes.pyi new file mode 100644 index 0000000000000000000000000000000000000000..e27843e5338213713e26973127c738c14313ff98 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/docutils/parsers/rst/nodes.pyi @@ -0,0 +1,3 @@ +from typing import Any + +def __getattr__(name: str) -> Any: ... diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/docutils/parsers/rst/roles.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/docutils/parsers/rst/roles.pyi new file mode 100644 index 0000000000000000000000000000000000000000..2c3d65b68c949b9bbbf70904f096a5c4df00287f --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/docutils/parsers/rst/roles.pyi @@ -0,0 +1,12 @@ +from typing import Any, Callable, Dict, List, Tuple + +import docutils.nodes +import docutils.parsers.rst.states + +_RoleFn = Callable[ + [str, str, str, int, docutils.parsers.rst.states.Inliner, Dict[str, Any], List[str]], + Tuple[List[docutils.nodes.reference], List[docutils.nodes.reference]], +] + +def register_local_role(name: str, role_fn: _RoleFn) -> None: ... +def __getattr__(name: str) -> Any: ... # incomplete diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/docutils/parsers/rst/states.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/docutils/parsers/rst/states.pyi new file mode 100644 index 0000000000000000000000000000000000000000..ac0872693442b8a85e61cf3f6a1cfcd220316563 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/docutils/parsers/rst/states.pyi @@ -0,0 +1,6 @@ +from typing import Any + +class Inliner: + def __init__(self) -> None: ... + +def __getattr__(name: str) -> Any: ... diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/freezegun/__init__.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/freezegun/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..1de0916f79df748b8784ee5f4e6a6e561293ef70 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/freezegun/__init__.pyi @@ -0,0 +1 @@ +from .api import freeze_time as freeze_time diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/freezegun/api.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/freezegun/api.pyi new file mode 100644 index 0000000000000000000000000000000000000000..a0d50ed7f49df409cbf19b3874d534f5d178e2c7 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/freezegun/api.pyi @@ -0,0 +1,56 @@ +from datetime import date, datetime, timedelta +from numbers import Real +from typing import Any, Awaitable, Callable, Iterator, Optional, Sequence, Type, TypeVar, Union, overload + +_T = TypeVar("_T") +_Freezable = Union[str, datetime, date, timedelta] + +class TickingDateTimeFactory(object): + def __init__(self, time_to_freeze: datetime, start: datetime) -> None: ... + def __call__(self) -> datetime: ... + +class FrozenDateTimeFactory(object): + def __init__(self, time_to_freeze: datetime) -> None: ... + def __call__(self) -> datetime: ... + def tick(self, delta: Union[float, Real, timedelta] = ...) -> None: ... + def move_to(self, target_datetime: Optional[_Freezable]) -> None: ... + +class StepTickTimeFactory(object): + def __init__(self, time_to_freeze: datetime, step_width: float) -> None: ... + def __call__(self) -> datetime: ... + def tick(self, delta: Optional[timedelta] = ...) -> None: ... + def update_step_width(self, step_width: float) -> None: ... + def move_to(self, target_datetime: Optional[_Freezable]) -> None: ... + +class _freeze_time: + def __init__( + self, + time_to_freeze_str: Optional[_Freezable], + tz_offset: float, + ignore: Sequence[str], + tick: bool, + as_arg: bool, + auto_tick_seconds: float, + ) -> None: ... + @overload + def __call__(self, func: Type[_T]) -> Type[_T]: ... + @overload + def __call__(self, func: Callable[..., Awaitable[_T]]) -> Callable[..., Awaitable[_T]]: ... + @overload + def __call__(self, func: Callable[..., _T]) -> Callable[..., _T]: ... + def __enter__(self) -> Any: ... + def __exit__(self, *args: Any) -> None: ... + def start(self) -> Any: ... + def stop(self) -> None: ... + def decorate_class(self, klass: Type[_T]) -> _T: ... + def decorate_coroutine(self, coroutine: _T) -> _T: ... + def decorate_callable(self, func: Callable[..., _T]) -> Callable[..., _T]: ... + +def freeze_time( + time_to_freeze: Optional[Union[_Freezable, Callable[..., _Freezable], Iterator[_Freezable]]] = ..., + tz_offset: Optional[float] = ..., + ignore: Optional[Sequence[str]] = ..., + tick: Optional[bool] = ..., + as_arg: Optional[bool] = ..., + auto_tick_seconds: Optional[float] = ..., +) -> _freeze_time: ... diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/frozendict.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/frozendict.pyi new file mode 100644 index 0000000000000000000000000000000000000000..704a44c135c8ae6b5226540250de750dc1cd66b2 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/frozendict.pyi @@ -0,0 +1,27 @@ +import collections +from typing import Any, Dict, Generic, Iterable, Iterator, Mapping, Tuple, Type, TypeVar, overload + +_S = TypeVar("_S") +_KT = TypeVar("_KT") +_VT = TypeVar("_VT") + +class frozendict(Mapping[_KT, _VT], Generic[_KT, _VT]): + + dict_cls: Type[Dict] = ... + @overload + def __init__(self, **kwargs: _VT) -> None: ... + @overload + def __init__(self, mapping: Mapping[_KT, _VT]) -> None: ... + @overload + def __init__(self, iterable: Iterable[Tuple[_KT, _VT]]) -> None: ... + def __getitem__(self, key: _KT) -> _VT: ... + def __contains__(self, key: object) -> bool: ... + def copy(self: _S, **add_or_replace: _VT) -> _S: ... + def __iter__(self) -> Iterator[_KT]: ... + def __len__(self) -> int: ... + def __repr__(self) -> str: ... + def __hash__(self) -> int: ... + +class FrozenOrderedDict(frozendict): + + dict_cls: Type[collections.OrderedDict] = ... diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/jwt/__init__.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/jwt/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..d2b45a2de6ceafff854a79e7051cffbc94ee1855 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/jwt/__init__.pyi @@ -0,0 +1,49 @@ +from typing import Any, Dict, Mapping, Optional, Union + +from cryptography.hazmat.primitives.asymmetric import rsa + +from . import algorithms + +def decode( + jwt: Union[str, bytes], + key: Union[str, bytes, rsa.RSAPublicKey, rsa.RSAPrivateKey] = ..., + verify: bool = ..., + algorithms: Optional[Any] = ..., + options: Optional[Mapping[Any, Any]] = ..., + **kwargs: Any, +) -> Dict[str, Any]: ... +def encode( + payload: Mapping[str, Any], + key: Union[str, bytes, rsa.RSAPublicKey, rsa.RSAPrivateKey], + algorithm: str = ..., + headers: Optional[Mapping[str, Any]] = ..., + json_encoder: Optional[Any] = ..., +) -> bytes: ... +def register_algorithm(alg_id: str, alg_obj: algorithms.Algorithm[Any]) -> None: ... +def unregister_algorithm(alg_id: str) -> None: ... + +class PyJWTError(Exception): ... +class InvalidTokenError(PyJWTError): ... +class DecodeError(InvalidTokenError): ... +class ExpiredSignatureError(InvalidTokenError): ... +class InvalidAudienceError(InvalidTokenError): ... +class InvalidIssuerError(InvalidTokenError): ... +class InvalidIssuedAtError(InvalidTokenError): ... +class ImmatureSignatureError(InvalidTokenError): ... +class InvalidKeyError(PyJWTError): ... +class InvalidAlgorithmError(InvalidTokenError): ... +class MissingRequiredClaimError(InvalidTokenError): ... +class InvalidSignatureError(DecodeError): ... + +# Compatibility aliases (deprecated) +ExpiredSignature = ExpiredSignatureError +InvalidAudience = InvalidAudienceError +InvalidIssuer = InvalidIssuerError + +# These aren't actually documented, but the package +# exports them in __init__.py, so we should at least +# make sure that mypy doesn't raise spurious errors +# if they're used. +get_unverified_header: Any +PyJWT: Any +PyJWS: Any diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/jwt/algorithms.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/jwt/algorithms.pyi new file mode 100644 index 0000000000000000000000000000000000000000..fe83e56b6be6d8605c473d05cfabbd3468697a32 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/jwt/algorithms.pyi @@ -0,0 +1,101 @@ +import sys +from hashlib import _Hash +from typing import Any, ClassVar, Dict, Generic, Optional, Set, TypeVar, Union + +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric.ec import ( + EllipticCurvePrivateKey, + EllipticCurvePrivateKeyWithSerialization, + EllipticCurvePrivateNumbers, + EllipticCurvePublicKey, + EllipticCurvePublicKeyWithSerialization, + EllipticCurvePublicNumbers, +) +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey +from cryptography.hazmat.primitives.asymmetric.rsa import ( + RSAPrivateKey, + RSAPrivateKeyWithSerialization, + RSAPrivateNumbers, + RSAPublicKey, + RSAPublicKeyWithSerialization, + RSAPublicNumbers, +) +from cryptography.hazmat.primitives.asymmetric.utils import Prehashed +from cryptography.hazmat.primitives.hashes import HashAlgorithm + +requires_cryptography = Set[str] + +def get_default_algorithms() -> Dict[str, Algorithm[Any]]: ... + +_K = TypeVar("_K") + +class Algorithm(Generic[_K]): + def prepare_key(self, key: _K) -> _K: ... + def sign(self, msg: bytes, key: _K) -> bytes: ... + def verify(self, msg: bytes, key: _K, sig: bytes) -> bool: ... + @staticmethod + def to_jwk(key_obj: _K) -> str: ... + @staticmethod + def from_jwk(jwk: str) -> _K: ... + +class NoneAlgorithm(Algorithm[None]): + def prepare_key(self, key: Optional[str]) -> None: ... + +class _HashAlg: + def __call__(self, arg: Union[bytes, bytearray, memoryview] = ...) -> _Hash: ... + +class HMACAlgorithm(Algorithm[bytes]): + SHA256: ClassVar[_HashAlg] + SHA384: ClassVar[_HashAlg] + SHA512: ClassVar[_HashAlg] + hash_alg: _HashAlg + def __init__(self, hash_alg: _HashAlg) -> None: ... + def prepare_key(self, key: Union[str, bytes]) -> bytes: ... + @staticmethod + def to_jwk(key_obj: Union[str, bytes]) -> str: ... + @staticmethod + def from_jwk(jwk: Union[str, bytes]) -> bytes: ... + +# Only defined if cryptography is installed. +class RSAAlgorithm(Algorithm[Any]): + SHA256: ClassVar[hashes.SHA256] + SHA384: ClassVar[hashes.SHA384] + SHA512: ClassVar[hashes.SHA512] + hash_alg: Union[HashAlgorithm, Prehashed] + def __init__(self, hash_alg: Union[HashAlgorithm, Prehashed]) -> None: ... + def prepare_key(self, key: Union[bytes, str, RSAPrivateKey, RSAPublicKey]) -> Union[RSAPrivateKey, RSAPublicKey]: ... + @staticmethod + def from_jwk(jwk: Union[str, bytes, Dict[str, Any]]) -> Union[RSAPrivateKey, RSAPublicKey]: ... + def sign(self, msg: bytes, key: RSAPrivateKey) -> bytes: ... + def verify(self, msg: bytes, key: RSAPublicKey, sig: bytes) -> bool: ... + +# Only defined if cryptography is installed. +class ECAlgorithm(Algorithm[Any]): + SHA256: ClassVar[hashes.SHA256] + SHA384: ClassVar[hashes.SHA384] + SHA512: ClassVar[hashes.SHA512] + hash_alg: Union[HashAlgorithm, Prehashed] + def __init__(self, hash_alg: Union[HashAlgorithm, Prehashed]) -> None: ... + def prepare_key( + self, key: Union[bytes, str, EllipticCurvePrivateKey, EllipticCurvePublicKey] + ) -> Union[EllipticCurvePrivateKey, EllipticCurvePublicKey]: ... + @staticmethod + def to_jwk(key_obj: Union[EllipticCurvePrivateKeyWithSerialization, EllipticCurvePublicKeyWithSerialization]) -> str: ... + @staticmethod + def from_jwk(jwk: Union[str, bytes]) -> Union[EllipticCurvePrivateKey, EllipticCurvePublicKey]: ... + def sign(self, msg: bytes, key: EllipticCurvePrivateKey) -> bytes: ... + def verify(self, msg: bytes, key: EllipticCurvePublicKey, sig: bytes) -> bool: ... + +# Only defined if cryptography is installed. Types should be tightened when +# cryptography gets type hints. +# See https://github.com/python/typeshed/issues/2542 +class RSAPSSAlgorithm(RSAAlgorithm): + def sign(self, msg: bytes, key: Any) -> bytes: ... + def verify(self, msg: bytes, key: Any, sig: bytes) -> bool: ... + +# Only defined if cryptography is installed. +class Ed25519Algorithm(Algorithm[Any]): + def __init__(self, **kwargs: Any) -> None: ... + def prepare_key(self, key: Union[str, bytes, Ed25519PrivateKey, Ed25519PublicKey]) -> Any: ... + def sign(self, msg: Union[str, bytes], key: Ed25519PrivateKey) -> bytes: ... + def verify(self, msg: Union[str, bytes], key: Ed25519PublicKey, sig: Union[str, bytes]) -> bool: ... diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/jwt/contrib/__init__.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/jwt/contrib/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/jwt/contrib/algorithms/__init__.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/jwt/contrib/algorithms/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/jwt/contrib/algorithms/py_ecdsa.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/jwt/contrib/algorithms/py_ecdsa.pyi new file mode 100644 index 0000000000000000000000000000000000000000..0f63de0a6cd0dcc21db8b1a6116260c443cd70a1 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/jwt/contrib/algorithms/py_ecdsa.pyi @@ -0,0 +1,10 @@ +import hashlib +from typing import Any + +from jwt.algorithms import Algorithm + +class ECAlgorithm(Algorithm[Any]): + SHA256: hashlib._Hash + SHA384: hashlib._Hash + SHA512: hashlib._Hash + def __init__(self, hash_alg: hashlib._Hash) -> None: ... diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/jwt/contrib/algorithms/pycrypto.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/jwt/contrib/algorithms/pycrypto.pyi new file mode 100644 index 0000000000000000000000000000000000000000..077684c67734f55a2c55bf58d0e3ae36e1f99e8c --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/jwt/contrib/algorithms/pycrypto.pyi @@ -0,0 +1,10 @@ +import hashlib +from typing import Any + +from jwt.algorithms import Algorithm + +class RSAAlgorithm(Algorithm[Any]): + SHA256: hashlib._Hash + SHA384: hashlib._Hash + SHA512: hashlib._Hash + def __init__(self, hash_alg: hashlib._Hash) -> None: ... diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/orjson.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/orjson.pyi new file mode 100644 index 0000000000000000000000000000000000000000..3fcbd7b668ce295ef6fd06369b84a433c0b8020a --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/orjson.pyi @@ -0,0 +1,24 @@ +from typing import Any, Callable, Optional, Union + +__version__: str + +def dumps(__obj: Any, default: Optional[Callable[[Any], Any]] = ..., option: Optional[int] = ...) -> bytes: ... +def loads(__obj: Union[bytes, bytearray, str]) -> Any: ... + +class JSONDecodeError(ValueError): ... +class JSONEncodeError(TypeError): ... + +OPT_APPEND_NEWLINE: int +OPT_INDENT_2: int +OPT_NAIVE_UTC: int +OPT_NON_STR_KEYS: int +OPT_OMIT_MICROSECONDS: int +OPT_PASSTHROUGH_DATACLASS: int +OPT_PASSTHROUGH_DATETIME: int +OPT_PASSTHROUGH_SUBCLASS: int +OPT_SERIALIZE_DATACLASS: int +OPT_SERIALIZE_NUMPY: int +OPT_SERIALIZE_UUID: int +OPT_SORT_KEYS: int +OPT_STRICT_INTEGER: int +OPT_UTC_Z: int diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/pkg_resources/__init__.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/pkg_resources/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..7f827747eac8d7983c0420164cbc42ab9cbd208f --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/pkg_resources/__init__.pyi @@ -0,0 +1,271 @@ +import importlib.abc +import types +import zipimport +from abc import ABCMeta +from typing import IO, Any, Callable, Dict, Generator, Iterable, List, Optional, Sequence, Set, Tuple, TypeVar, Union, overload + +_T = TypeVar("_T") +_NestedStr = Union[str, Iterable[Union[str, Iterable[Any]]]] +_InstallerType = Callable[[Requirement], Optional[Distribution]] +_EPDistType = Union[Distribution, Requirement, str] +_MetadataType = Optional[IResourceProvider] +_PkgReqType = Union[str, Requirement] +_DistFinderType = Callable[[_Importer, str, bool], Generator[Distribution, None, None]] +_NSHandlerType = Callable[[_Importer, str, str, types.ModuleType], str] + +def declare_namespace(name: str) -> None: ... +def fixup_namespace_packages(path_item: str) -> None: ... + +class WorkingSet: + entries: List[str] + def __init__(self, entries: Optional[Iterable[str]] = ...) -> None: ... + def require(self, *requirements: _NestedStr) -> Sequence[Distribution]: ... + def run_script(self, requires: str, script_name: str) -> None: ... + def iter_entry_points(self, group: str, name: Optional[str] = ...) -> Generator[EntryPoint, None, None]: ... + def add_entry(self, entry: str) -> None: ... + def __contains__(self, dist: Distribution) -> bool: ... + def __iter__(self) -> Generator[Distribution, None, None]: ... + def find(self, req: Requirement) -> Optional[Distribution]: ... + def resolve( + self, requirements: Iterable[Requirement], env: Optional[Environment] = ..., installer: Optional[_InstallerType] = ... + ) -> List[Distribution]: ... + def add(self, dist: Distribution, entry: Optional[str] = ..., insert: bool = ..., replace: bool = ...) -> None: ... + def subscribe(self, callback: Callable[[Distribution], None]) -> None: ... + def find_plugins( + self, plugin_env: Environment, full_env: Optional[Environment] = ..., fallback: bool = ... + ) -> Tuple[List[Distribution], Dict[Distribution, Exception]]: ... + +working_set: WorkingSet + +def require(*requirements: _NestedStr) -> Sequence[Distribution]: ... +def run_script(requires: str, script_name: str) -> None: ... +def iter_entry_points(group: str, name: Optional[str] = ...) -> Generator[EntryPoint, None, None]: ... +def add_activation_listener(callback: Callable[[Distribution], None]) -> None: ... + +class Environment: + def __init__( + self, search_path: Optional[Sequence[str]] = ..., platform: Optional[str] = ..., python: Optional[str] = ... + ) -> None: ... + def __getitem__(self, project_name: str) -> List[Distribution]: ... + def __iter__(self) -> Generator[str, None, None]: ... + def add(self, dist: Distribution) -> None: ... + def remove(self, dist: Distribution) -> None: ... + def can_add(self, dist: Distribution) -> bool: ... + def __add__(self, other: Union[Distribution, Environment]) -> Environment: ... + def __iadd__(self, other: Union[Distribution, Environment]) -> Environment: ... + @overload + def best_match(self, req: Requirement, working_set: WorkingSet) -> Distribution: ... + @overload + def best_match(self, req: Requirement, working_set: WorkingSet, installer: Callable[[Requirement], _T] = ...) -> _T: ... + @overload + def obtain(self, requirement: Requirement) -> None: ... + @overload + def obtain(self, requirement: Requirement, installer: Callable[[Requirement], _T] = ...) -> _T: ... + def scan(self, search_path: Optional[Sequence[str]] = ...) -> None: ... + +def parse_requirements(strs: Union[str, Iterable[str]]) -> Generator[Requirement, None, None]: ... + +class Requirement: + unsafe_name: str + project_name: str + key: str + extras: Tuple[str, ...] + specs: List[Tuple[str, str]] + # TODO: change this to Optional[packaging.markers.Marker] once we can import + # packaging.markers + marker: Optional[Any] + @staticmethod + def parse(s: Union[str, Iterable[str]]) -> Requirement: ... + def __contains__(self, item: Union[Distribution, str, Tuple[str, ...]]) -> bool: ... + def __eq__(self, other_requirement: Any) -> bool: ... + +def load_entry_point(dist: _EPDistType, group: str, name: str) -> Any: ... +def get_entry_info(dist: _EPDistType, group: str, name: str) -> Optional[EntryPoint]: ... +@overload +def get_entry_map(dist: _EPDistType) -> Dict[str, Dict[str, EntryPoint]]: ... +@overload +def get_entry_map(dist: _EPDistType, group: str) -> Dict[str, EntryPoint]: ... + +class EntryPoint: + name: str + module_name: str + attrs: Tuple[str, ...] + extras: Tuple[str, ...] + dist: Optional[Distribution] + def __init__( + self, + name: str, + module_name: str, + attrs: Tuple[str, ...] = ..., + extras: Tuple[str, ...] = ..., + dist: Optional[Distribution] = ..., + ) -> None: ... + @classmethod + def parse(cls, src: str, dist: Optional[Distribution] = ...) -> EntryPoint: ... + @classmethod + def parse_group( + cls, group: str, lines: Union[str, Sequence[str]], dist: Optional[Distribution] = ... + ) -> Dict[str, EntryPoint]: ... + @classmethod + def parse_map( + cls, data: Union[Dict[str, Union[str, Sequence[str]]], str, Sequence[str]], dist: Optional[Distribution] = ... + ) -> Dict[str, EntryPoint]: ... + def load(self, require: bool = ..., env: Optional[Environment] = ..., installer: Optional[_InstallerType] = ...) -> Any: ... + def require(self, env: Optional[Environment] = ..., installer: Optional[_InstallerType] = ...) -> None: ... + def resolve(self) -> Any: ... + +def find_distributions(path_item: str, only: bool = ...) -> Generator[Distribution, None, None]: ... +def get_distribution(dist: Union[Requirement, str, Distribution]) -> Distribution: ... + +class Distribution(IResourceProvider, IMetadataProvider): + PKG_INFO: str + location: str + project_name: str + key: str + extras: List[str] + version: str + parsed_version: Tuple[str, ...] + py_version: str + platform: Optional[str] + precedence: int + def __init__( + self, + location: Optional[str] = ..., + metadata: _MetadataType = ..., + project_name: Optional[str] = ..., + version: Optional[str] = ..., + py_version: str = ..., + platform: Optional[str] = ..., + precedence: int = ..., + ) -> None: ... + @classmethod + def from_location( + cls, location: str, basename: str, metadata: _MetadataType = ..., **kw: Union[str, None, int] + ) -> Distribution: ... + @classmethod + def from_filename(cls, filename: str, metadata: _MetadataType = ..., **kw: Union[str, None, int]) -> Distribution: ... + def activate(self, path: Optional[List[str]] = ...) -> None: ... + def as_requirement(self) -> Requirement: ... + def requires(self, extras: Tuple[str, ...] = ...) -> List[Requirement]: ... + def clone(self, **kw: Union[str, int, None]) -> Requirement: ... + def egg_name(self) -> str: ... + def __cmp__(self, other: Any) -> bool: ... + def get_entry_info(self, group: str, name: str) -> Optional[EntryPoint]: ... + @overload + def get_entry_map(self) -> Dict[str, Dict[str, EntryPoint]]: ... + @overload + def get_entry_map(self, group: str) -> Dict[str, EntryPoint]: ... + def load_entry_point(self, group: str, name: str) -> Any: ... + +EGG_DIST: int +BINARY_DIST: int +SOURCE_DIST: int +CHECKOUT_DIST: int +DEVELOP_DIST: int + +def resource_exists(package_or_requirement: _PkgReqType, resource_name: str) -> bool: ... +def resource_stream(package_or_requirement: _PkgReqType, resource_name: str) -> IO[bytes]: ... +def resource_string(package_or_requirement: _PkgReqType, resource_name: str) -> bytes: ... +def resource_isdir(package_or_requirement: _PkgReqType, resource_name: str) -> bool: ... +def resource_listdir(package_or_requirement: _PkgReqType, resource_name: str) -> List[str]: ... +def resource_filename(package_or_requirement: _PkgReqType, resource_name: str) -> str: ... +def set_extraction_path(path: str) -> None: ... +def cleanup_resources(force: bool = ...) -> List[str]: ... + +class IResourceManager: + def resource_exists(self, package_or_requirement: _PkgReqType, resource_name: str) -> bool: ... + def resource_stream(self, package_or_requirement: _PkgReqType, resource_name: str) -> IO[bytes]: ... + def resource_string(self, package_or_requirement: _PkgReqType, resource_name: str) -> bytes: ... + def resource_isdir(self, package_or_requirement: _PkgReqType, resource_name: str) -> bool: ... + def resource_listdir(self, package_or_requirement: _PkgReqType, resource_name: str) -> List[str]: ... + def resource_filename(self, package_or_requirement: _PkgReqType, resource_name: str) -> str: ... + def set_extraction_path(self, path: str) -> None: ... + def cleanup_resources(self, force: bool = ...) -> List[str]: ... + def get_cache_path(self, archive_name: str, names: Iterable[str] = ...) -> str: ... + def extraction_error(self) -> None: ... + def postprocess(self, tempname: str, filename: str) -> None: ... + +@overload +def get_provider(package_or_requirement: str) -> IResourceProvider: ... +@overload +def get_provider(package_or_requirement: Requirement) -> Distribution: ... + +class IMetadataProvider: + def has_metadata(self, name: str) -> bool: ... + def metadata_isdir(self, name: str) -> bool: ... + def metadata_listdir(self, name: str) -> List[str]: ... + def get_metadata(self, name: str) -> str: ... + def get_metadata_lines(self, name: str) -> Generator[str, None, None]: ... + def run_script(self, script_name: str, namespace: Dict[str, Any]) -> None: ... + +class ResolutionError(Exception): ... + +class DistributionNotFound(ResolutionError): + @property + def req(self) -> Requirement: ... + @property + def requirers(self) -> Set[str]: ... + @property + def requirers_str(self) -> str: ... + def report(self) -> str: ... + +class VersionConflict(ResolutionError): + @property + def dist(self) -> Any: ... + @property + def req(self) -> Any: ... + def report(self) -> str: ... + def with_context(self, required_by: Set[Union[Distribution, str]]) -> VersionConflict: ... + +class ContextualVersionConflict(VersionConflict): + @property + def required_by(self) -> Set[Union[Distribution, str]]: ... + +class UnknownExtra(ResolutionError): ... + +class ExtractionError(Exception): + manager: IResourceManager + cache_path: str + original_error: Exception + +class _Importer(importlib.abc.MetaPathFinder, importlib.abc.InspectLoader, metaclass=ABCMeta): ... + +def register_finder(importer_type: type, distribution_finder: _DistFinderType) -> None: ... +def register_loader_type(loader_type: type, provider_factory: Callable[[types.ModuleType], IResourceProvider]) -> None: ... +def register_namespace_handler(importer_type: type, namespace_handler: _NSHandlerType) -> None: ... + +class IResourceProvider(IMetadataProvider): ... +class NullProvider: ... +class EggProvider(NullProvider): ... +class DefaultProvider(EggProvider): ... + +class PathMetadata(DefaultProvider, IResourceProvider): + def __init__(self, path: str, egg_info: str) -> None: ... + +class ZipProvider(EggProvider): ... + +class EggMetadata(ZipProvider, IResourceProvider): + def __init__(self, zipimporter: zipimport.zipimporter) -> None: ... + +class EmptyProvider(NullProvider): ... + +empty_provider: EmptyProvider + +class FileMetadata(EmptyProvider, IResourceProvider): + def __init__(self, path_to_pkg_info: str) -> None: ... + +def parse_version(v: str) -> Tuple[str, ...]: ... +def yield_lines(strs: _NestedStr) -> Generator[str, None, None]: ... +def split_sections(strs: _NestedStr) -> Generator[Tuple[Optional[str], str], None, None]: ... +def safe_name(name: str) -> str: ... +def safe_version(version: str) -> str: ... +def safe_extra(extra: str) -> str: ... +def to_filename(name_or_version: str) -> str: ... +def get_build_platform() -> str: ... +def get_platform() -> str: ... +def get_supported_platform() -> str: ... +def compatible_platforms(provided: Optional[str], required: Optional[str]) -> bool: ... +def get_default_cache() -> str: ... +def get_importer(path_item: str) -> _Importer: ... +def ensure_directory(path: str) -> None: ... +def normalize_path(filename: str) -> str: ... diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/pkg_resources/py31compat.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/pkg_resources/py31compat.pyi new file mode 100644 index 0000000000000000000000000000000000000000..162da65e0e67604a9ddff219d27585df9ead3078 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/pkg_resources/py31compat.pyi @@ -0,0 +1,5 @@ +import os + +needs_makedirs: bool + +makedirs = os.makedirs diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/pyrfc3339/__init__.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/pyrfc3339/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..d8c1ca5e88c7765707a24a40378fc8695c7e6e65 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/pyrfc3339/__init__.pyi @@ -0,0 +1,2 @@ +from .generator import generate as generate +from .parser import parse as parse diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/pyrfc3339/generator.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/pyrfc3339/generator.pyi new file mode 100644 index 0000000000000000000000000000000000000000..9375b5b6110964a30627f3bb55fc3f30da449e64 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/pyrfc3339/generator.pyi @@ -0,0 +1,3 @@ +from datetime import datetime + +def generate(dt: datetime, utc: bool = ..., accept_naive: bool = ..., microseconds: bool = ...) -> str: ... diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/pyrfc3339/parser.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/pyrfc3339/parser.pyi new file mode 100644 index 0000000000000000000000000000000000000000..445b542c38cd2e203959fcdf531d338783700880 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/pyrfc3339/parser.pyi @@ -0,0 +1,3 @@ +from datetime import datetime + +def parse(timestamp: str, utc: bool = ..., produce_naive: bool = ...) -> datetime: ... diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/pyrfc3339/utils.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/pyrfc3339/utils.pyi new file mode 100644 index 0000000000000000000000000000000000000000..f49a8c06850240dcf62890a42f9e40fcedc3857d --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/pyrfc3339/utils.pyi @@ -0,0 +1,11 @@ +from datetime import datetime, timedelta, tzinfo +from typing import Optional + +class FixedOffset(tzinfo): + def __init__(self, hours: float, minutes: float) -> None: ... + def dst(self, dt: Optional[datetime]) -> timedelta: ... + def utcoffset(self, dt: Optional[datetime]) -> timedelta: ... + def tzname(self, dt: Optional[datetime]) -> str: ... + +def timedelta_seconds(td: timedelta) -> int: ... +def timezone(utcoffset: float) -> str: ... diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/__init__.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..bd055bbca80a9768bf336a55a945c9f73635950f --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/__init__.pyi @@ -0,0 +1,125 @@ +from __future__ import print_function + +import types +import typing +import unittest +from builtins import next as next +from functools import wraps as wraps +from io import BytesIO as BytesIO, StringIO as StringIO +from typing import ( + Any, + AnyStr, + Callable, + Dict, + ItemsView, + Iterable, + KeysView, + Mapping, + NoReturn, + Optional, + Pattern, + Tuple, + Type, + TypeVar, + Union, + ValuesView, + overload, +) + +from . import moves as moves + +_T = TypeVar("_T") +_K = TypeVar("_K") +_V = TypeVar("_V") + +__version__: str + +# TODO make constant, then move this stub to 2and3 +# https://github.com/python/typeshed/issues/17 +PY2 = False +PY3 = True +PY34: bool + +string_types = (str,) +integer_types = (int,) +class_types = (type,) +text_type = str +binary_type = bytes + +MAXSIZE: int + +def callable(obj: object) -> bool: ... +def get_unbound_function(unbound: types.FunctionType) -> types.FunctionType: ... +def create_bound_method(func: types.FunctionType, obj: object) -> types.MethodType: ... +def create_unbound_method(func: types.FunctionType, cls: type) -> types.FunctionType: ... + +Iterator = object + +def get_method_function(meth: types.MethodType) -> types.FunctionType: ... +def get_method_self(meth: types.MethodType) -> Optional[object]: ... +def get_function_closure(fun: types.FunctionType) -> Optional[Tuple[types._Cell, ...]]: ... +def get_function_code(fun: types.FunctionType) -> types.CodeType: ... +def get_function_defaults(fun: types.FunctionType) -> Optional[Tuple[Any, ...]]: ... +def get_function_globals(fun: types.FunctionType) -> Dict[str, Any]: ... +def iterkeys(d: Mapping[_K, _V]) -> typing.Iterator[_K]: ... +def itervalues(d: Mapping[_K, _V]) -> typing.Iterator[_V]: ... +def iteritems(d: Mapping[_K, _V]) -> typing.Iterator[Tuple[_K, _V]]: ... + +# def iterlists + +def viewkeys(d: Mapping[_K, _V]) -> KeysView[_K]: ... +def viewvalues(d: Mapping[_K, _V]) -> ValuesView[_V]: ... +def viewitems(d: Mapping[_K, _V]) -> ItemsView[_K, _V]: ... +def b(s: str) -> binary_type: ... +def u(s: str) -> text_type: ... + +unichr = chr + +def int2byte(i: int) -> bytes: ... +def byte2int(bs: binary_type) -> int: ... +def indexbytes(buf: binary_type, i: int) -> int: ... +def iterbytes(buf: binary_type) -> typing.Iterator[int]: ... +def assertCountEqual(self: unittest.TestCase, first: Iterable[_T], second: Iterable[_T], msg: Optional[str] = ...) -> None: ... +@overload +def assertRaisesRegex(self: unittest.TestCase, msg: Optional[str] = ...) -> Any: ... +@overload +def assertRaisesRegex(self: unittest.TestCase, callable_obj: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: ... +def assertRegex( + self: unittest.TestCase, text: AnyStr, expected_regex: Union[AnyStr, Pattern[AnyStr]], msg: Optional[str] = ... +) -> None: ... + +exec_ = exec + +def reraise( + tp: Optional[Type[BaseException]], value: Optional[BaseException], tb: Optional[types.TracebackType] = ... +) -> NoReturn: ... +def raise_from(value: Union[BaseException, Type[BaseException]], from_value: Optional[BaseException]) -> NoReturn: ... + +print_ = print + +def with_metaclass(meta: type, *bases: type) -> type: ... +def add_metaclass(metaclass: type) -> Callable[[_T], _T]: ... +def ensure_binary(s: Union[bytes, str], encoding: str = ..., errors: str = ...) -> bytes: ... +def ensure_str(s: Union[bytes, str], encoding: str = ..., errors: str = ...) -> str: ... +def ensure_text(s: Union[bytes, str], encoding: str = ..., errors: str = ...) -> str: ... +def python_2_unicode_compatible(klass: _T) -> _T: ... + +class _LazyDescriptor: + name: str + def __init__(self, name: str) -> None: ... + def __get__(self, obj: Optional[object], type: Optional[type] = ...) -> Any: ... + +class MovedModule(_LazyDescriptor): + mod: str + def __init__(self, name: str, old: str, new: Optional[str] = ...) -> None: ... + def __getattr__(self, attr: str) -> Any: ... + +class MovedAttribute(_LazyDescriptor): + mod: str + attr: str + def __init__( + self, name: str, old_mod: str, new_mod: str, old_attr: Optional[str] = ..., new_attr: Optional[str] = ... + ) -> None: ... + +def add_move(move: Union[MovedModule, MovedAttribute]) -> None: ... +def remove_move(name: str) -> None: ... diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/BaseHTTPServer.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/BaseHTTPServer.pyi new file mode 100644 index 0000000000000000000000000000000000000000..0e1ad713145883b4743b74f9a2732db47fecd4a5 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/BaseHTTPServer.pyi @@ -0,0 +1 @@ +from http.server import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/CGIHTTPServer.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/CGIHTTPServer.pyi new file mode 100644 index 0000000000000000000000000000000000000000..0e1ad713145883b4743b74f9a2732db47fecd4a5 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/CGIHTTPServer.pyi @@ -0,0 +1 @@ +from http.server import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/SimpleHTTPServer.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/SimpleHTTPServer.pyi new file mode 100644 index 0000000000000000000000000000000000000000..0e1ad713145883b4743b74f9a2732db47fecd4a5 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/SimpleHTTPServer.pyi @@ -0,0 +1 @@ +from http.server import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/__init__.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..3455676db51ebc5c0b1a461564e404d334f08831 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/__init__.pyi @@ -0,0 +1,65 @@ +# Stubs for six.moves +# +# Note: Commented out items means they weren't implemented at the time. +# Uncomment them when the modules have been added to the typeshed. +import importlib +import shlex +from builtins import filter as filter, input as input, map as map, range as range, zip as zip +from collections import UserDict as UserDict, UserList as UserList, UserString as UserString +from functools import reduce as reduce +from io import StringIO as StringIO +from itertools import filterfalse as filterfalse, zip_longest as zip_longest +from os import getcwd as getcwd, getcwdb as getcwdb +from sys import intern as intern + +# import tkinter.font as tkinter_font +# import tkinter.messagebox as tkinter_messagebox +# import tkinter.simpledialog as tkinter_tksimpledialog +# import tkinter.dnd as tkinter_dnd +# import tkinter.colorchooser as tkinter_colorchooser +# import tkinter.scrolledtext as tkinter_scrolledtext +# import tkinter.simpledialog as tkinter_simpledialog +# import tkinter.tix as tkinter_tix +# import copyreg as copyreg +# import dbm.gnu as dbm_gnu +from . import ( + BaseHTTPServer as BaseHTTPServer, + CGIHTTPServer as CGIHTTPServer, + SimpleHTTPServer as SimpleHTTPServer, + _dummy_thread as _dummy_thread, + _thread as _thread, + builtins as builtins, + configparser as configparser, + cPickle as cPickle, + email_mime_base as email_mime_base, + email_mime_multipart as email_mime_multipart, + email_mime_nonmultipart as email_mime_nonmultipart, + email_mime_text as email_mime_text, + html_entities as html_entities, + html_parser as html_parser, + http_client as http_client, + http_cookiejar as http_cookiejar, + http_cookies as http_cookies, + queue as queue, + reprlib as reprlib, + socketserver as socketserver, + tkinter as tkinter, + tkinter_commondialog as tkinter_commondialog, + tkinter_constants as tkinter_constants, + tkinter_dialog as tkinter_dialog, + tkinter_filedialog as tkinter_filedialog, + tkinter_tkfiledialog as tkinter_tkfiledialog, + tkinter_ttk as tkinter_ttk, + urllib as urllib, + urllib_error as urllib_error, + urllib_parse as urllib_parse, + urllib_robotparser as urllib_robotparser, +) + +# import xmlrpc.client as xmlrpc_client +# import xmlrpc.server as xmlrpc_server + +xrange = range +reload_module = importlib.reload +cStringIO = StringIO +shlex_quote = shlex.quote diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/_dummy_thread.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/_dummy_thread.pyi new file mode 100644 index 0000000000000000000000000000000000000000..24879612a769bed06e07ce438182b5127365eee9 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/_dummy_thread.pyi @@ -0,0 +1 @@ +from _dummy_thread import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/_thread.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/_thread.pyi new file mode 100644 index 0000000000000000000000000000000000000000..25952a61494f5bfd1d8590b0d8c0bc6463b2a557 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/_thread.pyi @@ -0,0 +1 @@ +from _thread import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/builtins.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/builtins.pyi new file mode 100644 index 0000000000000000000000000000000000000000..9596ba032aa04297e0de1b55984edc6644bdcf8e --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/builtins.pyi @@ -0,0 +1 @@ +from builtins import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/cPickle.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/cPickle.pyi new file mode 100644 index 0000000000000000000000000000000000000000..2b944b59d656595cf65b49208c94952ead33a51e --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/cPickle.pyi @@ -0,0 +1 @@ +from pickle import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/collections_abc.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/collections_abc.pyi new file mode 100644 index 0000000000000000000000000000000000000000..dba0f1535768ef4cb3b6ccee1a244f15efdf6d7a --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/collections_abc.pyi @@ -0,0 +1 @@ +from collections.abc import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/configparser.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/configparser.pyi new file mode 100644 index 0000000000000000000000000000000000000000..044861ce0c38d4b7901e7ac1970747b0628c5928 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/configparser.pyi @@ -0,0 +1 @@ +from configparser import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/email_mime_base.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/email_mime_base.pyi new file mode 100644 index 0000000000000000000000000000000000000000..4df155c939d5592c4301cf93099d7163c7f11619 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/email_mime_base.pyi @@ -0,0 +1 @@ +from email.mime.base import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/email_mime_multipart.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/email_mime_multipart.pyi new file mode 100644 index 0000000000000000000000000000000000000000..4f312412bbc0d41f6a645da66df4beefc14f224c --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/email_mime_multipart.pyi @@ -0,0 +1 @@ +from email.mime.multipart import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/email_mime_nonmultipart.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/email_mime_nonmultipart.pyi new file mode 100644 index 0000000000000000000000000000000000000000..c15c8c0440b5e4817384341f18351d5a078ba237 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/email_mime_nonmultipart.pyi @@ -0,0 +1 @@ +from email.mime.nonmultipart import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/email_mime_text.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/email_mime_text.pyi new file mode 100644 index 0000000000000000000000000000000000000000..51e147387fa218e3269958debaae49027f390e43 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/email_mime_text.pyi @@ -0,0 +1 @@ +from email.mime.text import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/html_entities.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/html_entities.pyi new file mode 100644 index 0000000000000000000000000000000000000000..c1244ddbee455e79f8eec1f16fb591a33965fd4d --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/html_entities.pyi @@ -0,0 +1 @@ +from html.entities import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/html_parser.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/html_parser.pyi new file mode 100644 index 0000000000000000000000000000000000000000..6db6dd83f35ee3d637c2235e7a2a6ec41571a073 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/html_parser.pyi @@ -0,0 +1 @@ +from html.parser import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/http_client.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/http_client.pyi new file mode 100644 index 0000000000000000000000000000000000000000..36d29b9551bc7014616d3355f86bd7cc614b68d1 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/http_client.pyi @@ -0,0 +1 @@ +from http.client import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/http_cookiejar.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/http_cookiejar.pyi new file mode 100644 index 0000000000000000000000000000000000000000..88a1aed6cc0f32642ea2ec2dff27830ba6d941e7 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/http_cookiejar.pyi @@ -0,0 +1 @@ +from http.cookiejar import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/http_cookies.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/http_cookies.pyi new file mode 100644 index 0000000000000000000000000000000000000000..9c59a53977bcaabfbb84049a1aa82d4bd8aeb4b2 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/http_cookies.pyi @@ -0,0 +1 @@ +from http.cookies import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/queue.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/queue.pyi new file mode 100644 index 0000000000000000000000000000000000000000..fe7be53a37ad9af2008cd02c63efc5ede8887ab3 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/queue.pyi @@ -0,0 +1 @@ +from queue import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/reprlib.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/reprlib.pyi new file mode 100644 index 0000000000000000000000000000000000000000..c329846fde0bb4bc67f2a25795d871e8162cf668 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/reprlib.pyi @@ -0,0 +1 @@ +from reprlib import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/socketserver.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/socketserver.pyi new file mode 100644 index 0000000000000000000000000000000000000000..6101c8bb022b079e1952cf245c41354bcc1691bb --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/socketserver.pyi @@ -0,0 +1 @@ +from socketserver import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/tkinter.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/tkinter.pyi new file mode 100644 index 0000000000000000000000000000000000000000..fc4d53a5a0b8410c2e1efd625058b95cc8afb38d --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/tkinter.pyi @@ -0,0 +1 @@ +from tkinter import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/tkinter_commondialog.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/tkinter_commondialog.pyi new file mode 100644 index 0000000000000000000000000000000000000000..34eb419619969ba060d50c0e44f90d0d052e3451 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/tkinter_commondialog.pyi @@ -0,0 +1 @@ +from tkinter.commondialog import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/tkinter_constants.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/tkinter_constants.pyi new file mode 100644 index 0000000000000000000000000000000000000000..3c04f6d84fa36b18f26bb031516b6a05eaf68b5e --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/tkinter_constants.pyi @@ -0,0 +1 @@ +from tkinter.constants import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/tkinter_dialog.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/tkinter_dialog.pyi new file mode 100644 index 0000000000000000000000000000000000000000..0da73c27acb8fac490dd3e5749f99115f51d91d7 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/tkinter_dialog.pyi @@ -0,0 +1 @@ +from tkinter.dialog import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/tkinter_filedialog.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/tkinter_filedialog.pyi new file mode 100644 index 0000000000000000000000000000000000000000..c4cc7c48b51cb4201fad2894abb37d156f084cef --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/tkinter_filedialog.pyi @@ -0,0 +1 @@ +from tkinter.filedialog import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/tkinter_tkfiledialog.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/tkinter_tkfiledialog.pyi new file mode 100644 index 0000000000000000000000000000000000000000..c4cc7c48b51cb4201fad2894abb37d156f084cef --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/tkinter_tkfiledialog.pyi @@ -0,0 +1 @@ +from tkinter.filedialog import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/tkinter_ttk.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/tkinter_ttk.pyi new file mode 100644 index 0000000000000000000000000000000000000000..14576f61c12c1f0bc00762c7e2e2ee7bdf25d940 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/tkinter_ttk.pyi @@ -0,0 +1 @@ +from tkinter.ttk import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/urllib_error.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/urllib_error.pyi new file mode 100644 index 0000000000000000000000000000000000000000..272007222364de8a82ad47aa8e79bb0fc290d4ad --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/urllib_error.pyi @@ -0,0 +1 @@ +from urllib.error import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/urllib_parse.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/urllib_parse.pyi new file mode 100644 index 0000000000000000000000000000000000000000..b557bbbb645f50d1615fb814322f683a5239d5ab --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/urllib_parse.pyi @@ -0,0 +1 @@ +from urllib.parse import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/urllib_request.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/urllib_request.pyi new file mode 100644 index 0000000000000000000000000000000000000000..dc03dcecc118d59ef2c8362d7b717792c9ed9c61 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/urllib_request.pyi @@ -0,0 +1 @@ +from .urllib.request import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/urllib_response.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/urllib_response.pyi new file mode 100644 index 0000000000000000000000000000000000000000..bbee52256e31ac2431edb921aa42c10dd8b4373b --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/urllib_response.pyi @@ -0,0 +1 @@ +from .urllib.response import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/urllib_robotparser.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/urllib_robotparser.pyi new file mode 100644 index 0000000000000000000000000000000000000000..bbf5c3ce4cd1aa62eb1cb891f2fb1d7b1432079f --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/six/moves/urllib_robotparser.pyi @@ -0,0 +1 @@ +from urllib.robotparser import * diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/typed_ast/__init__.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/typed_ast/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/typed_ast/ast27.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/typed_ast/ast27.pyi new file mode 100644 index 0000000000000000000000000000000000000000..139e58aac246e19330ef8f623312438eca685329 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/typed_ast/ast27.pyi @@ -0,0 +1,335 @@ +import typing +from typing import Any, Iterator, Optional, Union + +class NodeVisitor: + def visit(self, node: AST) -> Any: ... + def generic_visit(self, node: AST) -> None: ... + +class NodeTransformer(NodeVisitor): + def generic_visit(self, node: AST) -> None: ... + +def parse(source: Union[str, bytes], filename: Union[str, bytes] = ..., mode: str = ...) -> AST: ... +def copy_location(new_node: AST, old_node: AST) -> AST: ... +def dump(node: AST, annotate_fields: bool = ..., include_attributes: bool = ...) -> str: ... +def fix_missing_locations(node: AST) -> AST: ... +def get_docstring(node: AST, clean: bool = ...) -> Optional[bytes]: ... +def increment_lineno(node: AST, n: int = ...) -> AST: ... +def iter_child_nodes(node: AST) -> Iterator[AST]: ... +def iter_fields(node: AST) -> Iterator[typing.Tuple[str, Any]]: ... +def literal_eval(node_or_string: Union[str, AST]) -> Any: ... +def walk(node: AST) -> Iterator[AST]: ... + +PyCF_ONLY_AST: int + +# ast classes + +identifier = str + +class AST: + _attributes: typing.Tuple[str, ...] + _fields: typing.Tuple[str, ...] + def __init__(self, *args: Any, **kwargs: Any) -> None: ... + +class mod(AST): ... + +class Module(mod): + body: typing.List[stmt] + type_ignores: typing.List[TypeIgnore] + +class Interactive(mod): + body: typing.List[stmt] + +class Expression(mod): + body: expr + +class FunctionType(mod): + argtypes: typing.List[expr] + returns: expr + +class Suite(mod): + body: typing.List[stmt] + +class stmt(AST): + lineno: int + col_offset: int + +class FunctionDef(stmt): + name: identifier + args: arguments + body: typing.List[stmt] + decorator_list: typing.List[expr] + type_comment: Optional[str] + +class ClassDef(stmt): + name: identifier + bases: typing.List[expr] + body: typing.List[stmt] + decorator_list: typing.List[expr] + +class Return(stmt): + value: Optional[expr] + +class Delete(stmt): + targets: typing.List[expr] + +class Assign(stmt): + targets: typing.List[expr] + value: expr + type_comment: Optional[str] + +class AugAssign(stmt): + target: expr + op: operator + value: expr + +class Print(stmt): + dest: Optional[expr] + values: typing.List[expr] + nl: bool + +class For(stmt): + target: expr + iter: expr + body: typing.List[stmt] + orelse: typing.List[stmt] + type_comment: Optional[str] + +class While(stmt): + test: expr + body: typing.List[stmt] + orelse: typing.List[stmt] + +class If(stmt): + test: expr + body: typing.List[stmt] + orelse: typing.List[stmt] + +class With(stmt): + context_expr: expr + optional_vars: Optional[expr] + body: typing.List[stmt] + type_comment: Optional[str] + +class Raise(stmt): + type: Optional[expr] + inst: Optional[expr] + tback: Optional[expr] + +class TryExcept(stmt): + body: typing.List[stmt] + handlers: typing.List[ExceptHandler] + orelse: typing.List[stmt] + +class TryFinally(stmt): + body: typing.List[stmt] + finalbody: typing.List[stmt] + +class Assert(stmt): + test: expr + msg: Optional[expr] + +class Import(stmt): + names: typing.List[alias] + +class ImportFrom(stmt): + module: Optional[identifier] + names: typing.List[alias] + level: Optional[int] + +class Exec(stmt): + body: expr + globals: Optional[expr] + locals: Optional[expr] + +class Global(stmt): + names: typing.List[identifier] + +class Expr(stmt): + value: expr + +class Pass(stmt): ... +class Break(stmt): ... +class Continue(stmt): ... +class slice(AST): ... + +_slice = slice # this lets us type the variable named 'slice' below + +class Slice(slice): + lower: Optional[expr] + upper: Optional[expr] + step: Optional[expr] + +class ExtSlice(slice): + dims: typing.List[slice] + +class Index(slice): + value: expr + +class Ellipsis(slice): ... + +class expr(AST): + lineno: int + col_offset: int + +class BoolOp(expr): + op: boolop + values: typing.List[expr] + +class BinOp(expr): + left: expr + op: operator + right: expr + +class UnaryOp(expr): + op: unaryop + operand: expr + +class Lambda(expr): + args: arguments + body: expr + +class IfExp(expr): + test: expr + body: expr + orelse: expr + +class Dict(expr): + keys: typing.List[expr] + values: typing.List[expr] + +class Set(expr): + elts: typing.List[expr] + +class ListComp(expr): + elt: expr + generators: typing.List[comprehension] + +class SetComp(expr): + elt: expr + generators: typing.List[comprehension] + +class DictComp(expr): + key: expr + value: expr + generators: typing.List[comprehension] + +class GeneratorExp(expr): + elt: expr + generators: typing.List[comprehension] + +class Yield(expr): + value: Optional[expr] + +class Compare(expr): + left: expr + ops: typing.List[cmpop] + comparators: typing.List[expr] + +class Call(expr): + func: expr + args: typing.List[expr] + keywords: typing.List[keyword] + starargs: Optional[expr] + kwargs: Optional[expr] + +class Repr(expr): + value: expr + +class Num(expr): + n: Union[int, float, complex] + +class Str(expr): + s: Union[str, bytes] + kind: str + +class Attribute(expr): + value: expr + attr: identifier + ctx: expr_context + +class Subscript(expr): + value: expr + slice: _slice + ctx: expr_context + +class Name(expr): + id: identifier + ctx: expr_context + +class List(expr): + elts: typing.List[expr] + ctx: expr_context + +class Tuple(expr): + elts: typing.List[expr] + ctx: expr_context + +class expr_context(AST): ... +class AugLoad(expr_context): ... +class AugStore(expr_context): ... +class Del(expr_context): ... +class Load(expr_context): ... +class Param(expr_context): ... +class Store(expr_context): ... +class boolop(AST): ... +class And(boolop): ... +class Or(boolop): ... +class operator(AST): ... +class Add(operator): ... +class BitAnd(operator): ... +class BitOr(operator): ... +class BitXor(operator): ... +class Div(operator): ... +class FloorDiv(operator): ... +class LShift(operator): ... +class Mod(operator): ... +class Mult(operator): ... +class Pow(operator): ... +class RShift(operator): ... +class Sub(operator): ... +class unaryop(AST): ... +class Invert(unaryop): ... +class Not(unaryop): ... +class UAdd(unaryop): ... +class USub(unaryop): ... +class cmpop(AST): ... +class Eq(cmpop): ... +class Gt(cmpop): ... +class GtE(cmpop): ... +class In(cmpop): ... +class Is(cmpop): ... +class IsNot(cmpop): ... +class Lt(cmpop): ... +class LtE(cmpop): ... +class NotEq(cmpop): ... +class NotIn(cmpop): ... + +class comprehension(AST): + target: expr + iter: expr + ifs: typing.List[expr] + +class ExceptHandler(AST): + type: Optional[expr] + name: Optional[expr] + body: typing.List[stmt] + lineno: int + col_offset: int + +class arguments(AST): + args: typing.List[expr] + vararg: Optional[identifier] + kwarg: Optional[identifier] + defaults: typing.List[expr] + type_comments: typing.List[Optional[str]] + +class keyword(AST): + arg: identifier + value: expr + +class alias(AST): + name: identifier + asname: Optional[identifier] + +class TypeIgnore(AST): + lineno: int diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/typed_ast/ast3.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/typed_ast/ast3.pyi new file mode 100644 index 0000000000000000000000000000000000000000..3dac9b2e1fd4fccb0244c956d66481f38466237a --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/typed_ast/ast3.pyi @@ -0,0 +1,384 @@ +import typing +from typing import Any, Iterator, Optional, Union + +class NodeVisitor: + def visit(self, node: AST) -> Any: ... + def generic_visit(self, node: AST) -> None: ... + +class NodeTransformer(NodeVisitor): + def generic_visit(self, node: AST) -> None: ... + +def parse(source: Union[str, bytes], filename: Union[str, bytes] = ..., mode: str = ..., feature_version: int = ...) -> AST: ... +def copy_location(new_node: AST, old_node: AST) -> AST: ... +def dump(node: AST, annotate_fields: bool = ..., include_attributes: bool = ...) -> str: ... +def fix_missing_locations(node: AST) -> AST: ... +def get_docstring(node: AST, clean: bool = ...) -> Optional[str]: ... +def increment_lineno(node: AST, n: int = ...) -> AST: ... +def iter_child_nodes(node: AST) -> Iterator[AST]: ... +def iter_fields(node: AST) -> Iterator[typing.Tuple[str, Any]]: ... +def literal_eval(node_or_string: Union[str, AST]) -> Any: ... +def walk(node: AST) -> Iterator[AST]: ... + +PyCF_ONLY_AST: int + +# ast classes + +identifier = str + +class AST: + _attributes: typing.Tuple[str, ...] + _fields: typing.Tuple[str, ...] + def __init__(self, *args: Any, **kwargs: Any) -> None: ... + +class mod(AST): ... + +class Module(mod): + body: typing.List[stmt] + type_ignores: typing.List[TypeIgnore] + +class Interactive(mod): + body: typing.List[stmt] + +class Expression(mod): + body: expr + +class FunctionType(mod): + argtypes: typing.List[expr] + returns: expr + +class Suite(mod): + body: typing.List[stmt] + +class stmt(AST): + lineno: int + col_offset: int + +class FunctionDef(stmt): + name: identifier + args: arguments + body: typing.List[stmt] + decorator_list: typing.List[expr] + returns: Optional[expr] + type_comment: Optional[str] + +class AsyncFunctionDef(stmt): + name: identifier + args: arguments + body: typing.List[stmt] + decorator_list: typing.List[expr] + returns: Optional[expr] + type_comment: Optional[str] + +class ClassDef(stmt): + name: identifier + bases: typing.List[expr] + keywords: typing.List[keyword] + body: typing.List[stmt] + decorator_list: typing.List[expr] + +class Return(stmt): + value: Optional[expr] + +class Delete(stmt): + targets: typing.List[expr] + +class Assign(stmt): + targets: typing.List[expr] + value: expr + type_comment: Optional[str] + +class AugAssign(stmt): + target: expr + op: operator + value: expr + +class AnnAssign(stmt): + target: expr + annotation: expr + value: Optional[expr] + simple: int + +class For(stmt): + target: expr + iter: expr + body: typing.List[stmt] + orelse: typing.List[stmt] + type_comment: Optional[str] + +class AsyncFor(stmt): + target: expr + iter: expr + body: typing.List[stmt] + orelse: typing.List[stmt] + type_comment: Optional[str] + +class While(stmt): + test: expr + body: typing.List[stmt] + orelse: typing.List[stmt] + +class If(stmt): + test: expr + body: typing.List[stmt] + orelse: typing.List[stmt] + +class With(stmt): + items: typing.List[withitem] + body: typing.List[stmt] + type_comment: Optional[str] + +class AsyncWith(stmt): + items: typing.List[withitem] + body: typing.List[stmt] + type_comment: Optional[str] + +class Raise(stmt): + exc: Optional[expr] + cause: Optional[expr] + +class Try(stmt): + body: typing.List[stmt] + handlers: typing.List[ExceptHandler] + orelse: typing.List[stmt] + finalbody: typing.List[stmt] + +class Assert(stmt): + test: expr + msg: Optional[expr] + +class Import(stmt): + names: typing.List[alias] + +class ImportFrom(stmt): + module: Optional[identifier] + names: typing.List[alias] + level: Optional[int] + +class Global(stmt): + names: typing.List[identifier] + +class Nonlocal(stmt): + names: typing.List[identifier] + +class Expr(stmt): + value: expr + +class Pass(stmt): ... +class Break(stmt): ... +class Continue(stmt): ... +class slice(AST): ... + +_slice = slice # this lets us type the variable named 'slice' below + +class Slice(slice): + lower: Optional[expr] + upper: Optional[expr] + step: Optional[expr] + +class ExtSlice(slice): + dims: typing.List[slice] + +class Index(slice): + value: expr + +class expr(AST): + lineno: int + col_offset: int + +class BoolOp(expr): + op: boolop + values: typing.List[expr] + +class BinOp(expr): + left: expr + op: operator + right: expr + +class UnaryOp(expr): + op: unaryop + operand: expr + +class Lambda(expr): + args: arguments + body: expr + +class IfExp(expr): + test: expr + body: expr + orelse: expr + +class Dict(expr): + keys: typing.List[expr] + values: typing.List[expr] + +class Set(expr): + elts: typing.List[expr] + +class ListComp(expr): + elt: expr + generators: typing.List[comprehension] + +class SetComp(expr): + elt: expr + generators: typing.List[comprehension] + +class DictComp(expr): + key: expr + value: expr + generators: typing.List[comprehension] + +class GeneratorExp(expr): + elt: expr + generators: typing.List[comprehension] + +class Await(expr): + value: expr + +class Yield(expr): + value: Optional[expr] + +class YieldFrom(expr): + value: expr + +class Compare(expr): + left: expr + ops: typing.List[cmpop] + comparators: typing.List[expr] + +class Call(expr): + func: expr + args: typing.List[expr] + keywords: typing.List[keyword] + +class Num(expr): + n: Union[float, int, complex] + +class Str(expr): + s: str + kind: str + +class FormattedValue(expr): + value: expr + conversion: typing.Optional[int] + format_spec: typing.Optional[expr] + +class JoinedStr(expr): + values: typing.List[expr] + +class Bytes(expr): + s: bytes + +class NameConstant(expr): + value: Any + +class Ellipsis(expr): ... + +class Attribute(expr): + value: expr + attr: identifier + ctx: expr_context + +class Subscript(expr): + value: expr + slice: _slice + ctx: expr_context + +class Starred(expr): + value: expr + ctx: expr_context + +class Name(expr): + id: identifier + ctx: expr_context + +class List(expr): + elts: typing.List[expr] + ctx: expr_context + +class Tuple(expr): + elts: typing.List[expr] + ctx: expr_context + +class expr_context(AST): ... +class AugLoad(expr_context): ... +class AugStore(expr_context): ... +class Del(expr_context): ... +class Load(expr_context): ... +class Param(expr_context): ... +class Store(expr_context): ... +class boolop(AST): ... +class And(boolop): ... +class Or(boolop): ... +class operator(AST): ... +class Add(operator): ... +class BitAnd(operator): ... +class BitOr(operator): ... +class BitXor(operator): ... +class Div(operator): ... +class FloorDiv(operator): ... +class LShift(operator): ... +class Mod(operator): ... +class Mult(operator): ... +class MatMult(operator): ... +class Pow(operator): ... +class RShift(operator): ... +class Sub(operator): ... +class unaryop(AST): ... +class Invert(unaryop): ... +class Not(unaryop): ... +class UAdd(unaryop): ... +class USub(unaryop): ... +class cmpop(AST): ... +class Eq(cmpop): ... +class Gt(cmpop): ... +class GtE(cmpop): ... +class In(cmpop): ... +class Is(cmpop): ... +class IsNot(cmpop): ... +class Lt(cmpop): ... +class LtE(cmpop): ... +class NotEq(cmpop): ... +class NotIn(cmpop): ... + +class comprehension(AST): + target: expr + iter: expr + ifs: typing.List[expr] + is_async: int + +class ExceptHandler(AST): + type: Optional[expr] + name: Optional[identifier] + body: typing.List[stmt] + lineno: int + col_offset: int + +class arguments(AST): + args: typing.List[arg] + vararg: Optional[arg] + kwonlyargs: typing.List[arg] + kw_defaults: typing.List[expr] + kwarg: Optional[arg] + defaults: typing.List[expr] + +class arg(AST): + arg: identifier + annotation: Optional[expr] + lineno: int + col_offset: int + type_comment: typing.Optional[str] + +class keyword(AST): + arg: Optional[identifier] + value: expr + +class alias(AST): + name: identifier + asname: Optional[identifier] + +class withitem(AST): + context_expr: expr + optional_vars: Optional[expr] + +class TypeIgnore(AST): + lineno: int diff --git a/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/typed_ast/conversions.pyi b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/typed_ast/conversions.pyi new file mode 100644 index 0000000000000000000000000000000000000000..c7088ecc2f7278574e61de30f5d3ae9ac236aee0 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/third_party/typeshed/third_party/3/typed_ast/conversions.pyi @@ -0,0 +1,3 @@ +from . import ast3, ast27 + +def py2to3(ast: ast27.AST) -> ast3.AST: ... diff --git a/lib/python3.12/site-packages/jedi/utils.py b/lib/python3.12/site-packages/jedi/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..dd529be7a936758d1b26421f1c8f05733fbd3308 --- /dev/null +++ b/lib/python3.12/site-packages/jedi/utils.py @@ -0,0 +1,134 @@ +""" +Utilities for end-users. +""" + +import __main__ +from collections import namedtuple +import logging +import traceback +import re +import os +import sys + +from jedi import Interpreter + + +READLINE_DEBUG = False + + +def setup_readline(namespace_module=__main__, fuzzy=False): + """ + This function sets up :mod:`readline` to use Jedi in a Python interactive + shell. + + If you want to use a custom ``PYTHONSTARTUP`` file (typically + ``$HOME/.pythonrc.py``), you can add this piece of code:: + + try: + from jedi.utils import setup_readline + except ImportError: + # Fallback to the stdlib readline completer if it is installed. + # Taken from http://docs.python.org/2/library/rlcompleter.html + print("Jedi is not installed, falling back to readline") + try: + import readline + import rlcompleter + readline.parse_and_bind("tab: complete") + except ImportError: + print("Readline is not installed either. No tab completion is enabled.") + else: + setup_readline() + + This will fallback to the readline completer if Jedi is not installed. + The readline completer will only complete names in the global namespace, + so for example:: + + ran + + will complete to ``range``. + + With Jedi the following code:: + + range(10).cou + + will complete to ``range(10).count``, this does not work with the default + cPython :mod:`readline` completer. + + You will also need to add ``export PYTHONSTARTUP=$HOME/.pythonrc.py`` to + your shell profile (usually ``.bash_profile`` or ``.profile`` if you use + bash). + """ + if READLINE_DEBUG: + logging.basicConfig( + filename='/tmp/jedi.log', + filemode='a', + level=logging.DEBUG + ) + + class JediRL: + def complete(self, text, state): + """ + This complete stuff is pretty weird, a generator would make + a lot more sense, but probably due to backwards compatibility + this is still the way how it works. + + The only important part is stuff in the ``state == 0`` flow, + everything else has been copied from the ``rlcompleter`` std. + library module. + """ + if state == 0: + sys.path.insert(0, os.getcwd()) + # Calling python doesn't have a path, so add to sys.path. + try: + logging.debug("Start REPL completion: " + repr(text)) + interpreter = Interpreter(text, [namespace_module.__dict__]) + + completions = interpreter.complete(fuzzy=fuzzy) + logging.debug("REPL completions: %s", completions) + + self.matches = [ + text[:len(text) - c._like_name_length] + c.name_with_symbols + for c in completions + ] + except: + logging.error("REPL Completion error:\n" + traceback.format_exc()) + raise + finally: + sys.path.pop(0) + try: + return self.matches[state] + except IndexError: + return None + + try: + # Need to import this one as well to make sure it's executed before + # this code. This didn't use to be an issue until 3.3. Starting with + # 3.4 this is different, it always overwrites the completer if it's not + # already imported here. + import rlcompleter # noqa: F401 + import readline + except ImportError: + print("Jedi: Module readline not available.") + else: + readline.set_completer(JediRL().complete) + readline.parse_and_bind("tab: complete") + # jedi itself does the case matching + readline.parse_and_bind("set completion-ignore-case on") + # because it's easier to hit the tab just once + readline.parse_and_bind("set show-all-if-unmodified") + readline.parse_and_bind("set show-all-if-ambiguous on") + # don't repeat all the things written in the readline all the time + readline.parse_and_bind("set completion-prefix-display-length 2") + # No delimiters, Jedi handles that. + readline.set_completer_delims('') + + +def version_info(): + """ + Returns a namedtuple of Jedi's version, similar to Python's + ``sys.version_info``. + """ + Version = namedtuple('Version', 'major, minor, micro') + from jedi import __version__ + tupl = re.findall(r'[a-z]+|\d+', __version__) + return Version(*[x if i == 3 else int(x) for i, x in enumerate(tupl)]) diff --git a/lib/python3.12/site-packages/pyzmq-26.4.0.dist-info/INSTALLER b/lib/python3.12/site-packages/pyzmq-26.4.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.12/site-packages/pyzmq-26.4.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.12/site-packages/pyzmq-26.4.0.dist-info/METADATA b/lib/python3.12/site-packages/pyzmq-26.4.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..957d3a49dd68fcd220e1fe721cd40f8cf1d58be1 --- /dev/null +++ b/lib/python3.12/site-packages/pyzmq-26.4.0.dist-info/METADATA @@ -0,0 +1,139 @@ +Metadata-Version: 2.2 +Name: pyzmq +Version: 26.4.0 +Summary: Python bindings for 0MQ +Author: Brian E. Granger, Min Ragan-Kelley +Author-Email: PyZMQ Contributors +License: BSD 3-Clause License + + Copyright (c) 2009-2012, Brian Granger, Min Ragan-Kelley + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: Science/Research +Classifier: Intended Audience :: System Administrators +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: MacOS :: MacOS X +Classifier: Operating System :: Microsoft :: Windows +Classifier: Operating System :: POSIX +Classifier: Topic :: System :: Networking +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Project-URL: Homepage, https://pyzmq.readthedocs.org +Project-URL: Documentation, https://pyzmq.readthedocs.org +Project-URL: Source, https://github.com/zeromq/pyzmq +Project-URL: Tracker, https://github.com/zeromq/pyzmq/issues +Requires-Python: >=3.8 +Requires-Dist: cffi; implementation_name == "pypy" +Description-Content-Type: text/markdown + +# PyZMQ: Python bindings for ØMQ + +This package contains Python bindings for [ZeroMQ](https://zeromq.org). +ØMQ is a lightweight and fast messaging implementation. + +PyZMQ should work with any reasonable version of Python (≥ 3.8), as well as PyPy. +PyZMQ supports libzmq ≥ 3.2.2 (including 4.x). + +For a summary of changes to pyzmq, see our +[changelog](https://pyzmq.readthedocs.io/en/latest/changelog.html). + +### ØMQ 3.x, 4.x + +PyZMQ fully supports the stable (not DRAFT) 3.x and 4.x APIs of libzmq, +developed at [zeromq/libzmq](https://github.com/zeromq/libzmq). +No code to change, no flags to pass, +just build pyzmq against the latest and it should work. + +## Documentation + +See PyZMQ's Sphinx-generated +documentation [on Read the Docs](https://pyzmq.readthedocs.io) for API +details, and some notes on Python and Cython development. If you want to +learn about using ØMQ in general, the excellent [ØMQ +Guide](http://zguide.zeromq.org/py:all) is the place to start, which has a +Python version of every example. We also have some information on our +[wiki](https://github.com/zeromq/pyzmq/wiki). + +## Downloading + +Unless you specifically want to develop PyZMQ, we recommend downloading +the PyZMQ source code or wheels from +[PyPI](https://pypi.io/project/pyzmq/), +or install with conda. + +You can also get the latest source code from our GitHub repository, but +building from the repository will require that you install recent Cython. + +## Building and installation + +For more detail on building pyzmq, see [our docs](https://pyzmq.readthedocs.io/en/latest/howto/build.html). + +We build wheels for macOS, Windows, and Linux, so you can get a binary on those platforms with: + +``` +pip install pyzmq +``` + +but compiling from source with `pip install pyzmq` should work in most environments. +Make sure you are using the latest pip, or it may not find the right wheels. + +If the wheel doesn't work for some reason, or you want to force pyzmq to be compiled +(this is often preferable if you already have libzmq installed and configured the way you want it), +you can force installation from source with: + +``` +pip install --no-binary=pyzmq pyzmq +``` + +## Old versions + +pyzmq 16 drops support Python 2.6 and 3.2. +If you need to use one of those Python versions, you can pin your pyzmq version to before 16: + +``` +pip install 'pyzmq<16' +``` + +For libzmq 2.0.x, use 'pyzmq\<2.1' + +pyzmq-2.1.11 was the last version of pyzmq to support Python 2.5, +and pyzmq ≥ 2.2.0 requires Python ≥ 2.6. +pyzmq-13.0.0 introduces PyPy support via CFFI, which only supports libzmq-3.2.2 and newer. + +PyZMQ releases ≤ 2.2.0 matched libzmq versioning, but this is no longer the case, +starting with PyZMQ 13.0.0 (it was the thirteenth release, so why not?). +PyZMQ ≥ 13.0 follows semantic versioning conventions accounting only for PyZMQ itself. diff --git a/lib/python3.12/site-packages/pyzmq-26.4.0.dist-info/RECORD b/lib/python3.12/site-packages/pyzmq-26.4.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..1f459db68617d104e5cd80a061121c67b2e0ae98 --- /dev/null +++ b/lib/python3.12/site-packages/pyzmq-26.4.0.dist-info/RECORD @@ -0,0 +1,165 @@ +pyzmq-26.4.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +pyzmq-26.4.0.dist-info/METADATA,sha256=wAVdH-9oQi6ZcV1CfODF803xuMfqvAl1uhUBs4CJpBg,6008 +pyzmq-26.4.0.dist-info/RECORD,, +pyzmq-26.4.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +pyzmq-26.4.0.dist-info/WHEEL,sha256=xzH26Gt9Qq9F-BV1MViXB9rWNxwQBFDqpbAdP7YC-ek,118 +pyzmq-26.4.0.dist-info/licenses/LICENSE.md,sha256=wM9fXAP41ncveicd8ctnEFRXi9PXlSfHL8Hyj4zHKno,1545 +pyzmq-26.4.0.dist-info/licenses/licenses/LICENSE.libsodium.txt,sha256=Q5ZNl2pts_uYavaJ0F-MoOmXGHi8yucJdQ2sj9xKmc8,823 +pyzmq-26.4.0.dist-info/licenses/licenses/LICENSE.tornado.txt,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358 +pyzmq-26.4.0.dist-info/licenses/licenses/LICENSE.zeromq.txt,sha256=HyVuytGSiAUQ6ErWBHTqt1iSGHhLmlC8fO7jTCuR8dU,16725 +pyzmq.libs/libsodium-14725a37.so.26.2.0,sha256=SKJ68geNDD-FTJkS-sZ1FMzDiOTR1Y7VD07_x5MOS_8,1209897 +pyzmq.libs/libzmq-56587b1b.so.5.2.5,sha256=iQXnc45s0ReytNoXSegEb3AAwzsU9lZflVoQGOcPX8I,950593 +zmq/__init__.pxd,sha256=P2y5B_9nDB_0RD7hxbXpVm4Jia1rKclTnnUVrSbF4lE,63 +zmq/__init__.py,sha256=0zUxdN9mC6mBJAOkBfsI4em8roHbvH0afHQTYsMFjXA,2232 +zmq/__init__.pyi,sha256=2Vk8hzDtjG-PtgybCgyA3B31dxk17IwtzkuXLrauLdc,922 +zmq/__pycache__/__init__.cpython-312.pyc,, +zmq/__pycache__/_future.cpython-312.pyc,, +zmq/__pycache__/_typing.cpython-312.pyc,, +zmq/__pycache__/asyncio.cpython-312.pyc,, +zmq/__pycache__/constants.cpython-312.pyc,, +zmq/__pycache__/decorators.cpython-312.pyc,, +zmq/__pycache__/error.cpython-312.pyc,, +zmq/_future.py,sha256=jLLA1bP3zC_JV6H1F3ZY7qO_URPaaCuAjb9vQ2e6Df0,24340 +zmq/_future.pyi,sha256=HLN20gfpZRfe9RMpW_kdsmnLrlxlO8r-W-TAuyWvZGM,3322 +zmq/_typing.py,sha256=FArL9c6FqYiEdVjv5Wf5-1nJnbZcD3KIjRm17mOxHrI,241 +zmq/asyncio.py,sha256=NjDmuZakTa2_4tvEH6Kn6j8s-bJ6JO0N7QscPdx_bfo,6518 +zmq/auth/__init__.py,sha256=D0XJjPJgN0ZqSBLDrbLm3l3N5pMQt75pv8LyearhsM8,346 +zmq/auth/__pycache__/__init__.cpython-312.pyc,, +zmq/auth/__pycache__/asyncio.cpython-312.pyc,, +zmq/auth/__pycache__/base.cpython-312.pyc,, +zmq/auth/__pycache__/certs.cpython-312.pyc,, +zmq/auth/__pycache__/ioloop.cpython-312.pyc,, +zmq/auth/__pycache__/thread.cpython-312.pyc,, +zmq/auth/asyncio.py,sha256=KLD0Kwev61dnImVhcLmEKr-PwTCqIyurWjs4SuH442A,1799 +zmq/auth/base.py,sha256=OPTB58nTeYJL-bu9bHa4lXUCCMwzo7cwlhxRuHBjd0Y,16337 +zmq/auth/certs.py,sha256=0lyPqG3o-ucI_UvCVpihxT10V9-hKJcyi5Us4trVGR0,4329 +zmq/auth/ioloop.py,sha256=xXF6P8A-HZlXfIYMVv8BW8EI_H2PjrJFF3a6Ajvd1rI,1298 +zmq/auth/thread.py,sha256=mv1NfTxJIydLhIkKdExPVDdWpcbczUDshIsYLUzhSkI,4103 +zmq/backend/__init__.py,sha256=5PfcIfpIzxGIegOaHbO3dBilBzby3l0yKC0o_Qf1m3A,940 +zmq/backend/__init__.pyi,sha256=yYO_6IRAThgQBnMviaA8ZSKtAb-B2y7wzotSDWT_hqk,3370 +zmq/backend/__pycache__/__init__.cpython-312.pyc,, +zmq/backend/__pycache__/select.cpython-312.pyc,, +zmq/backend/cffi/README.md,sha256=u7zNkS3dJALRPzpgPv5s4Q1tIkevm0BMzVpcwZD0PoM,95 +zmq/backend/cffi/__init__.py,sha256=wNVPr6F3_fGxvBjQclPfZizcn59Yhh2Z_20XwNw8JXk,898 +zmq/backend/cffi/__pycache__/__init__.cpython-312.pyc,, +zmq/backend/cffi/__pycache__/_poll.cpython-312.pyc,, +zmq/backend/cffi/__pycache__/context.cpython-312.pyc,, +zmq/backend/cffi/__pycache__/devices.cpython-312.pyc,, +zmq/backend/cffi/__pycache__/error.cpython-312.pyc,, +zmq/backend/cffi/__pycache__/message.cpython-312.pyc,, +zmq/backend/cffi/__pycache__/socket.cpython-312.pyc,, +zmq/backend/cffi/__pycache__/utils.cpython-312.pyc,, +zmq/backend/cffi/_cdefs.h,sha256=68YmiAPwDvk8Cyuxke3wOgWBzNzO6eMZx5d21V051vc,2641 +zmq/backend/cffi/_cffi_src.c,sha256=ADxHwdYLRjsRf8vuyOK8lbHiXTjgbfoiIwOjarFQ8ds,1314 +zmq/backend/cffi/_poll.py,sha256=niEuO85_fLbRntELWu0k89qLicJPgnirX13fTnw0Irc,2884 +zmq/backend/cffi/context.py,sha256=dKoVS0VJa0A1N3l9LYGRoYlRLmmHZigmnMhZyHsP-jA,1899 +zmq/backend/cffi/devices.py,sha256=r3sLr3lyrj0mjVx_NenHQAtA9BWyhsUGuJoCHmplLXg,1480 +zmq/backend/cffi/error.py,sha256=h-mZPwc1TydObeTU3XUmMi9KUeqs3NJYNTqUTkt5dyI,311 +zmq/backend/cffi/message.py,sha256=7AOjzkymUuPNNKu1wzY4qw09ex1xpy2Wy9M69km0G3Y,6675 +zmq/backend/cffi/socket.py,sha256=p9V461ba6ctz6AU98FPo4oeQcdGrd6mPuPDy-YqHTfA,12044 +zmq/backend/cffi/utils.py,sha256=fN1m75YzUgZCLhrFnlBWiwB-hS3g8ri9mxstMyU7dKY,2056 +zmq/backend/cython/__init__.pxd,sha256=iRgsrNY8-yEX3UL83jFHziSPaVibZx-qltTXcYVUM9Y,60 +zmq/backend/cython/__init__.py,sha256=SMuE8GAtsmEMWNGrv-ZlQZcuTD3EVU838E1ViRHLDNI,322 +zmq/backend/cython/__pycache__/__init__.cpython-312.pyc,, +zmq/backend/cython/__pycache__/_zmq.cpython-312.pyc,, +zmq/backend/cython/_externs.pxd,sha256=iVF0FfbYZU1qPYvhmi8IhMKBgVgjSCYWJsgjl0c6FTk,349 +zmq/backend/cython/_zmq.cpython-312-x86_64-linux-gnu.so,sha256=jP4Aa8dBAdFPNAp9BelmralGRu7qnJ8JlW-n89Awioo,303457 +zmq/backend/cython/_zmq.pxd,sha256=W1GDf_ESvs6tWBK6Xn8iu0GthDdUwAY6-d0-QDTMNWU,2282 +zmq/backend/cython/_zmq.py,sha256=5c4TRRif75_hc7IURMFzfVjZi4AFv0tlv21WwwBnglc,59522 +zmq/backend/cython/constant_enums.pxi,sha256=LNVbov9C6GBuJvWHnfpqUjmNT0x8alTeub885-o_mI0,7562 +zmq/backend/cython/libzmq.pxd,sha256=uT4VJldFxBXK9E0BWdPlRAwmH3kSIkVZeI-xrQIr_Pk,4490 +zmq/backend/select.py,sha256=wpFbaaYbWJn89NTzS9dPyum8JJdNgakEICTFLI7t-Ws,888 +zmq/constants.py,sha256=xEyRW8hA1lLjDAnMjOWvmKAFQqZYzaTWO62dA7atnbM,28341 +zmq/decorators.py,sha256=sLeTjxsNcnjKYCsyUsx5RyC0X2Sfqi355nvBDzLDxGY,5099 +zmq/devices/__init__.py,sha256=Ao9r2eNUSgj3We4Zv0CN54jRKsrVoLf9PXYCQQbLQZI,769 +zmq/devices/__pycache__/__init__.cpython-312.pyc,, +zmq/devices/__pycache__/basedevice.cpython-312.pyc,, +zmq/devices/__pycache__/monitoredqueue.cpython-312.pyc,, +zmq/devices/__pycache__/monitoredqueuedevice.cpython-312.pyc,, +zmq/devices/__pycache__/proxydevice.cpython-312.pyc,, +zmq/devices/__pycache__/proxysteerabledevice.cpython-312.pyc,, +zmq/devices/basedevice.py,sha256=Ma39_n1J6LzTrhTz3ZxoXZaniDksCpzGYPDhlK0nvPg,9542 +zmq/devices/monitoredqueue.py,sha256=au2EN-fFLXDvVRFClAYS3q2Lb-KxW3keT9l82W5BKRo,1294 +zmq/devices/monitoredqueuedevice.py,sha256=onjY-L74VC_YjppiC4C_voBgbuEtRhstdTaytxQe14I,1929 +zmq/devices/proxydevice.py,sha256=MCB4j-65vyjSLytrzEWfB2q6YZLn_035Sxfns9j4yyQ,2843 +zmq/devices/proxysteerabledevice.py,sha256=atHF4HRd7A_lQQV8q6eHzQ_BaT-Gv6D3EabnN1p67vQ,3206 +zmq/error.py,sha256=fkENA-HNylo18IcEUcZTZqeSEienKFNLKck8_l7QeCE,5343 +zmq/eventloop/__init__.py,sha256=j5PpZdLAwLtwChrGCEZHJYJ6ZJoEzNBMlzY9r5K5iUw,103 +zmq/eventloop/__pycache__/__init__.cpython-312.pyc,, +zmq/eventloop/__pycache__/_deprecated.cpython-312.pyc,, +zmq/eventloop/__pycache__/future.cpython-312.pyc,, +zmq/eventloop/__pycache__/ioloop.cpython-312.pyc,, +zmq/eventloop/__pycache__/zmqstream.cpython-312.pyc,, +zmq/eventloop/_deprecated.py,sha256=APinUSjjM4B3Bn2VlqXzR4OJj_RkyQbWJUDvjndStsY,6443 +zmq/eventloop/future.py,sha256=lueaaPliVxJkvTaksnmAJkd09XZUfi4o0YnAQiFsciI,2612 +zmq/eventloop/ioloop.py,sha256=pmFSoqjZUy40wbibneTYwyDfVy43a1Ffvzus3pdelU4,766 +zmq/eventloop/zmqstream.py,sha256=VTvYZnxJbT9uwHN2opzYQPcXT59divswpwcvU52NMCE,23041 +zmq/green/__init__.py,sha256=Vmg7Zv4rXt9dUbgy7pGx1a8igWFMqcIWRnRrzZq3Jx4,1367 +zmq/green/__pycache__/__init__.cpython-312.pyc,, +zmq/green/__pycache__/core.cpython-312.pyc,, +zmq/green/__pycache__/device.cpython-312.pyc,, +zmq/green/__pycache__/poll.cpython-312.pyc,, +zmq/green/core.py,sha256=CdD5t314XkL5ZcAszaaKdg3ceCSZjzeunPrX81_THTI,11447 +zmq/green/device.py,sha256=HTtdyyENo8aOtpklkoDnfSVJKbM5Xh_-KMJZbImZgSQ,978 +zmq/green/eventloop/__init__.py,sha256=N13sRnQlJDo2gD70qPNZP7uc_EEMAjE6hDa-SLhKj0s,68 +zmq/green/eventloop/__pycache__/__init__.cpython-312.pyc,, +zmq/green/eventloop/__pycache__/ioloop.cpython-312.pyc,, +zmq/green/eventloop/__pycache__/zmqstream.cpython-312.pyc,, +zmq/green/eventloop/ioloop.py,sha256=rNJvPZsF-SZpXFEk7T8DXUE5yMFxltF5HE9qZkCmufc,43 +zmq/green/eventloop/zmqstream.py,sha256=3LGGOp9Lx0OxrsiNNxt4jdzNAJvXZNMLdlOYcsrDz8c,291 +zmq/green/poll.py,sha256=77Jpd7h-TJ0ZLE7vm1J7hfJfRCm3BI41cL5CYmUzH1A,2996 +zmq/log/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +zmq/log/__main__.py,sha256=FsHekF9qqnfhDkGvl8zVWxUuckLxTqubxMr6GjuCyTA,4005 +zmq/log/__pycache__/__init__.cpython-312.pyc,, +zmq/log/__pycache__/__main__.cpython-312.pyc,, +zmq/log/__pycache__/handlers.cpython-312.pyc,, +zmq/log/handlers.py,sha256=PLdJvzN3J6pg1Z4mied6RxwoKoW6VF8vz1mivTMl74Y,7228 +zmq/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +zmq/ssh/__init__.py,sha256=2Wcr18a8hS5Qjwhn1p6SYb6NMDIi7Y48JRXg56iU0fI,29 +zmq/ssh/__pycache__/__init__.cpython-312.pyc,, +zmq/ssh/__pycache__/forward.cpython-312.pyc,, +zmq/ssh/__pycache__/tunnel.cpython-312.pyc,, +zmq/ssh/forward.py,sha256=m2p4E7GVUYOSZdGbNkGNjoq6LG-ZzLKlhE0sZVa4e1U,3318 +zmq/ssh/tunnel.py,sha256=SCurCVb4WkKFZluqDKQzMkTsmlM0SHRNno5fUNRkrUk,13357 +zmq/sugar/__init__.py,sha256=pNZlBhg8DnJPQoodyICE1zlPsABcpzDcipPVE3PTkw4,1061 +zmq/sugar/__init__.pyi,sha256=F_JYIucugCUuXik_FSVfzWXICyuH1yDzlshcZRb8bDU,219 +zmq/sugar/__pycache__/__init__.cpython-312.pyc,, +zmq/sugar/__pycache__/attrsettr.cpython-312.pyc,, +zmq/sugar/__pycache__/context.cpython-312.pyc,, +zmq/sugar/__pycache__/frame.cpython-312.pyc,, +zmq/sugar/__pycache__/poll.cpython-312.pyc,, +zmq/sugar/__pycache__/socket.cpython-312.pyc,, +zmq/sugar/__pycache__/stopwatch.cpython-312.pyc,, +zmq/sugar/__pycache__/tracker.cpython-312.pyc,, +zmq/sugar/__pycache__/version.cpython-312.pyc,, +zmq/sugar/attrsettr.py,sha256=LL3MjUFm2TW4VYTsmv1FwEATM3Qw_MGT6s9fB1mBVek,2638 +zmq/sugar/context.py,sha256=hGIxHjkE7urLIFDhuRXSIR9atKsBPXrkvtIEigrH8KA,14575 +zmq/sugar/frame.py,sha256=OpWj-JQypaq54XDpgfUtVSFTpJ7KPwwd6B562DrJmm8,4264 +zmq/sugar/poll.py,sha256=7qQnUTtQJL8IN5S2VeB0jheoqUKp37pM20yjt4Fv6P4,5752 +zmq/sugar/socket.py,sha256=Lf8ksPBLVBVNiq1it140DURWVt-1VmBlWnUsWWoANwc,35362 +zmq/sugar/stopwatch.py,sha256=i1Cg96aPzsiHmUTAZEgSsiZ5qQJ7rw-pFgiIYJoJU1g,935 +zmq/sugar/tracker.py,sha256=raZKyJc3SYxlY17uAQpIPkHUaNk7bt9cwtdYugLd8QQ,3603 +zmq/sugar/version.py,sha256=jFsFWOQ2cic2ULerOYh5aWW3u7eZVV9_pa0uNpDMHpM,1620 +zmq/tests/__init__.py,sha256=U0V-_B2wSbo4XGxJTQ6EFhInOstNXaHsUcF7Fj9BXok,7940 +zmq/tests/__pycache__/__init__.cpython-312.pyc,, +zmq/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +zmq/utils/__pycache__/__init__.cpython-312.pyc,, +zmq/utils/__pycache__/garbage.cpython-312.pyc,, +zmq/utils/__pycache__/interop.cpython-312.pyc,, +zmq/utils/__pycache__/jsonapi.cpython-312.pyc,, +zmq/utils/__pycache__/monitor.cpython-312.pyc,, +zmq/utils/__pycache__/strtypes.cpython-312.pyc,, +zmq/utils/__pycache__/win32.cpython-312.pyc,, +zmq/utils/__pycache__/z85.cpython-312.pyc,, +zmq/utils/garbage.py,sha256=hfBcYNhJum7TcW4ktHJWa7CoNnDvSRFc4K5IK7IguWY,6124 +zmq/utils/getpid_compat.h,sha256=emvckPfSlYeCoUNgfYTkAWC4ie-LXLRnXDNLlXxXaPI,116 +zmq/utils/interop.py,sha256=l4AsLmDz3UHmuHjwc5EEZ61P_56HGVIVg88Lj4wnjPk,685 +zmq/utils/ipcmaxlen.h,sha256=q-YGX5BECL_QpOzOx3oC_I8mcNCWbJJ6FnUrdKlG1fU,522 +zmq/utils/jsonapi.py,sha256=2G1kMc3EW_Y4jSH_DwZzSti8lxzpRTx02kLrDokVDSA,1025 +zmq/utils/monitor.py,sha256=YVSdus-G9fwDmrGt2KtFj6GJIdzidq3Yvp6FZOyQFQ4,3287 +zmq/utils/mutex.h,sha256=SU-nEHeFNc_mzxQcDb9c8NpdLJTLpyECpy1GyYFpOFE,1646 +zmq/utils/pyversion_compat.h,sha256=4FkQ95UVmA_as9lBrIO7-wM5D0tEinVAlYmZls_SRT0,284 +zmq/utils/strtypes.py,sha256=sd0-cJGuDntYAcBMO_uqWAwvsOJGp8WXudMz3nJRUUA,1376 +zmq/utils/win32.py,sha256=aBQFhfZuNXvkHEtiYx5cqCAl3Mkl0qjsRzloIwR2K90,4940 +zmq/utils/z85.py,sha256=AM_l4fxgzA823RETaKj1QR8ci3gto-FoqvRm4qzxDXI,1838 +zmq/utils/zmq_compat.h,sha256=MmhfsPdKBoeja8cRuKrtTyI6zPtZfOFTtQTdc3VhbAk,2339 diff --git a/lib/python3.12/site-packages/pyzmq-26.4.0.dist-info/REQUESTED b/lib/python3.12/site-packages/pyzmq-26.4.0.dist-info/REQUESTED new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/pyzmq-26.4.0.dist-info/WHEEL b/lib/python3.12/site-packages/pyzmq-26.4.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..f2cf813fd56da31ccacda630c50a651f508bbc3f --- /dev/null +++ b/lib/python3.12/site-packages/pyzmq-26.4.0.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: scikit-build-core 0.11.1 +Root-Is-Purelib: false +Tag: cp312-cp312-manylinux_2_28_x86_64 + diff --git a/lib/python3.12/site-packages/pyzmq-26.4.0.dist-info/licenses/LICENSE.md b/lib/python3.12/site-packages/pyzmq-26.4.0.dist-info/licenses/LICENSE.md new file mode 100644 index 0000000000000000000000000000000000000000..f7072d1c994fbb6eb8a48365be0d583cb02a03e9 --- /dev/null +++ b/lib/python3.12/site-packages/pyzmq-26.4.0.dist-info/licenses/LICENSE.md @@ -0,0 +1,30 @@ +BSD 3-Clause License + +Copyright (c) 2009-2012, Brian Granger, Min Ragan-Kelley + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/lib/python3.12/site-packages/pyzmq-26.4.0.dist-info/licenses/licenses/LICENSE.libsodium.txt b/lib/python3.12/site-packages/pyzmq-26.4.0.dist-info/licenses/licenses/LICENSE.libsodium.txt new file mode 100644 index 0000000000000000000000000000000000000000..6bce7f0581c80aa6d05d33dd5267e1e047c738a9 --- /dev/null +++ b/lib/python3.12/site-packages/pyzmq-26.4.0.dist-info/licenses/licenses/LICENSE.libsodium.txt @@ -0,0 +1,18 @@ +/* + * ISC License + * + * Copyright (c) 2013-2024 + * Frank Denis + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ diff --git a/lib/python3.12/site-packages/pyzmq-26.4.0.dist-info/licenses/licenses/LICENSE.tornado.txt b/lib/python3.12/site-packages/pyzmq-26.4.0.dist-info/licenses/licenses/LICENSE.tornado.txt new file mode 100644 index 0000000000000000000000000000000000000000..d645695673349e3947e8e5ae42332d0ac3164cd7 --- /dev/null +++ b/lib/python3.12/site-packages/pyzmq-26.4.0.dist-info/licenses/licenses/LICENSE.tornado.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/lib/python3.12/site-packages/pyzmq-26.4.0.dist-info/licenses/licenses/LICENSE.zeromq.txt b/lib/python3.12/site-packages/pyzmq-26.4.0.dist-info/licenses/licenses/LICENSE.zeromq.txt new file mode 100644 index 0000000000000000000000000000000000000000..a612ad9813b006ce81d1ee438dd784da99a54007 --- /dev/null +++ b/lib/python3.12/site-packages/pyzmq-26.4.0.dist-info/licenses/licenses/LICENSE.zeromq.txt @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/lib/python3.12/site-packages/torchvision-0.21.0.dist-info/INSTALLER b/lib/python3.12/site-packages/torchvision-0.21.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.12/site-packages/torchvision-0.21.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.12/site-packages/torchvision-0.21.0.dist-info/LICENSE b/lib/python3.12/site-packages/torchvision-0.21.0.dist-info/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..1edcf92c3317b90fedd187e2eaad101bd1c1efc5 --- /dev/null +++ b/lib/python3.12/site-packages/torchvision-0.21.0.dist-info/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) Soumith Chintala 2016, +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/lib/python3.12/site-packages/torchvision-0.21.0.dist-info/METADATA b/lib/python3.12/site-packages/torchvision-0.21.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..61d8ded27429db77f62a3e8c5144e4fd13e163fc --- /dev/null +++ b/lib/python3.12/site-packages/torchvision-0.21.0.dist-info/METADATA @@ -0,0 +1,147 @@ +Metadata-Version: 2.1 +Name: torchvision +Version: 0.21.0 +Summary: image and video datasets and models for torch deep learning +Home-page: https://github.com/pytorch/vision +Author: PyTorch Core Team +Author-email: soumith@pytorch.org +License: BSD +Requires-Python: >=3.9 +Description-Content-Type: text/markdown +License-File: LICENSE +Requires-Dist: numpy +Requires-Dist: torch (==2.6.0) +Requires-Dist: pillow (!=8.3.*,>=5.3.0) +Provides-Extra: gdown +Requires-Dist: gdown (>=4.7.3) ; extra == 'gdown' +Provides-Extra: scipy +Requires-Dist: scipy ; extra == 'scipy' + +# torchvision + +[![total torchvision downloads](https://pepy.tech/badge/torchvision)](https://pepy.tech/project/torchvision) +[![documentation](https://img.shields.io/badge/dynamic/json.svg?label=docs&url=https%3A%2F%2Fpypi.org%2Fpypi%2Ftorchvision%2Fjson&query=%24.info.version&colorB=brightgreen&prefix=v)](https://pytorch.org/vision/stable/index.html) + +The torchvision package consists of popular datasets, model architectures, and common image transformations for computer +vision. + +## Installation + +Please refer to the [official +instructions](https://pytorch.org/get-started/locally/) to install the stable +versions of `torch` and `torchvision` on your system. + +To build source, refer to our [contributing +page](https://github.com/pytorch/vision/blob/main/CONTRIBUTING.md#development-installation). + +The following is the corresponding `torchvision` versions and supported Python +versions. + +| `torch` | `torchvision` | Python | +| ------------------ | ------------------ | ------------------- | +| `main` / `nightly` | `main` / `nightly` | `>=3.9`, `<=3.12` | +| `2.5` | `0.20` | `>=3.9`, `<=3.12` | +| `2.4` | `0.19` | `>=3.8`, `<=3.12` | +| `2.3` | `0.18` | `>=3.8`, `<=3.12` | +| `2.2` | `0.17` | `>=3.8`, `<=3.11` | +| `2.1` | `0.16` | `>=3.8`, `<=3.11` | +| `2.0` | `0.15` | `>=3.8`, `<=3.11` | + +
+ older versions + +| `torch` | `torchvision` | Python | +|---------|-------------------|---------------------------| +| `1.13` | `0.14` | `>=3.7.2`, `<=3.10` | +| `1.12` | `0.13` | `>=3.7`, `<=3.10` | +| `1.11` | `0.12` | `>=3.7`, `<=3.10` | +| `1.10` | `0.11` | `>=3.6`, `<=3.9` | +| `1.9` | `0.10` | `>=3.6`, `<=3.9` | +| `1.8` | `0.9` | `>=3.6`, `<=3.9` | +| `1.7` | `0.8` | `>=3.6`, `<=3.9` | +| `1.6` | `0.7` | `>=3.6`, `<=3.8` | +| `1.5` | `0.6` | `>=3.5`, `<=3.8` | +| `1.4` | `0.5` | `==2.7`, `>=3.5`, `<=3.8` | +| `1.3` | `0.4.2` / `0.4.3` | `==2.7`, `>=3.5`, `<=3.7` | +| `1.2` | `0.4.1` | `==2.7`, `>=3.5`, `<=3.7` | +| `1.1` | `0.3` | `==2.7`, `>=3.5`, `<=3.7` | +| `<=1.0` | `0.2` | `==2.7`, `>=3.5`, `<=3.7` | + +
+ +## Image Backends + +Torchvision currently supports the following image backends: + +- torch tensors +- PIL images: + - [Pillow](https://python-pillow.org/) + - [Pillow-SIMD](https://github.com/uploadcare/pillow-simd) - a **much faster** drop-in replacement for Pillow with SIMD. + +Read more in in our [docs](https://pytorch.org/vision/stable/transforms.html). + +## [UNSTABLE] Video Backend + +Torchvision currently supports the following video backends: + +- [pyav](https://github.com/PyAV-Org/PyAV) (default) - Pythonic binding for ffmpeg libraries. +- video_reader - This needs ffmpeg to be installed and torchvision to be built from source. There shouldn't be any + conflicting version of ffmpeg installed. Currently, this is only supported on Linux. + +``` +conda install -c conda-forge 'ffmpeg<4.3' +python setup.py install +``` + +# Using the models on C++ + +Refer to [example/cpp](https://github.com/pytorch/vision/tree/main/examples/cpp). + +**DISCLAIMER**: the `libtorchvision` library includes the torchvision +custom ops as well as most of the C++ torchvision APIs. Those APIs do not come +with any backward-compatibility guarantees and may change from one version to +the next. Only the Python APIs are stable and with backward-compatibility +guarantees. So, if you need stability within a C++ environment, your best bet is +to export the Python APIs via torchscript. + +## Documentation + +You can find the API documentation on the pytorch website: + +## Contributing + +See the [CONTRIBUTING](CONTRIBUTING.md) file for how to help out. + +## Disclaimer on Datasets + +This is a utility library that downloads and prepares public datasets. We do not host or distribute these datasets, +vouch for their quality or fairness, or claim that you have license to use the dataset. It is your responsibility to +determine whether you have permission to use the dataset under the dataset's license. + +If you're a dataset owner and wish to update any part of it (description, citation, etc.), or do not want your dataset +to be included in this library, please get in touch through a GitHub issue. Thanks for your contribution to the ML +community! + +## Pre-trained Model License + +The pre-trained models provided in this library may have their own licenses or terms and conditions derived from the +dataset used for training. It is your responsibility to determine whether you have permission to use the models for your +use case. + +More specifically, SWAG models are released under the CC-BY-NC 4.0 license. See +[SWAG LICENSE](https://github.com/facebookresearch/SWAG/blob/main/LICENSE) for additional details. + +## Citing TorchVision + +If you find TorchVision useful in your work, please consider citing the following BibTeX entry: + +```bibtex +@software{torchvision2016, + title = {TorchVision: PyTorch's Computer Vision library}, + author = {TorchVision maintainers and contributors}, + year = 2016, + journal = {GitHub repository}, + publisher = {GitHub}, + howpublished = {\url{https://github.com/pytorch/vision}} +} +``` diff --git a/lib/python3.12/site-packages/torchvision-0.21.0.dist-info/RECORD b/lib/python3.12/site-packages/torchvision-0.21.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..636971865cc31c26d5ef5649745e3af74da5835a --- /dev/null +++ b/lib/python3.12/site-packages/torchvision-0.21.0.dist-info/RECORD @@ -0,0 +1,386 @@ +torchvision-0.21.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +torchvision-0.21.0.dist-info/LICENSE,sha256=ZQL2doUc_iX4r3VTHfsyN1tzJbc8N-e0N0H6QiiT5x0,1517 +torchvision-0.21.0.dist-info/METADATA,sha256=ysg7cjmHbnltbKQLf28tr2m4A3OFJVVClgfh0_bNeE0,6134 +torchvision-0.21.0.dist-info/RECORD,, +torchvision-0.21.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +torchvision-0.21.0.dist-info/WHEEL,sha256=Fa4luOZMtbP9T1lH9r5CAb1tiEKk2EoLvCD0sxktAMY,104 +torchvision-0.21.0.dist-info/top_level.txt,sha256=ucJZoaluBW9BGYT4TuCE6zoZY_JuSP30wbDh-IRpxUU,12 +torchvision.libs/libcudart.41118559.so.12,sha256=h3QiT1sRpzsV0HSj_M5zJzIsXEz9_ZJNaoJnee7JaP4,707904 +torchvision.libs/libjpeg.1c1c4b09.so.8,sha256=4g0_-nHW9O7mxv9bXLzFpOSE-qh6R-p3uvlk46XfJp8,664504 +torchvision.libs/libnvjpeg.02b6d700.so.12,sha256=btLqpaLN_zk7yVxUNHS5YRemcYi76kbqdNEvf4OHR5c,6722352 +torchvision.libs/libpng16.0364a1db.so.16,sha256=XHT13jO77EQJX8hxsCRwNo7ZG3J-g1CRsyCuZhAvxns,279617 +torchvision.libs/libsharpyuv.5c41a003.so.0,sha256=-eg4IlWNkqxcjPdLHw4P5KRbvCRIqde5D6KwU1VOCbw,33632 +torchvision.libs/libwebp.54a0d02a.so.7,sha256=G2OQ2SplenNV4UEOvCUpJ08szkGXY15doXitLl4Ej8U,712281 +torchvision.libs/libz.d13a2644.so.1,sha256=Cw5oKp3H_UiVpngyiPhRt5PciWM_KHFAJ5dPpNZvORQ,124744 +torchvision/_C.so,sha256=G8EaPzd7lXK6r5LForyBeCQowFqGWguBetPORHY_vj4,7761160 +torchvision/__init__.py,sha256=7iyfQRDPEgPbSMQmAWBzKawfGXCfqRwVL42V61NDenM,3534 +torchvision/__pycache__/__init__.cpython-312.pyc,, +torchvision/__pycache__/_internally_replaced_utils.cpython-312.pyc,, +torchvision/__pycache__/_meta_registrations.cpython-312.pyc,, +torchvision/__pycache__/_utils.cpython-312.pyc,, +torchvision/__pycache__/extension.cpython-312.pyc,, +torchvision/__pycache__/utils.cpython-312.pyc,, +torchvision/__pycache__/version.cpython-312.pyc,, +torchvision/_internally_replaced_utils.py,sha256=SmFV-P4ETuJBIbrFUV4s1rXSpP_FI7U72CPO6q-6mwA,1407 +torchvision/_meta_registrations.py,sha256=lkEGW61fKUrGSh0iOFsZ1ZHskItS1EJ9Oo2UfM-OvQ8,7208 +torchvision/_utils.py,sha256=6TWK0JGaZVQrofgCAp5ox61_NQE2gIwhYouKQMiTaJ8,934 +torchvision/datasets/__init__.py,sha256=CQfKDHOvjSZ2dBlOLRieEjP9AsB1jJEe3-5vbFNeXOM,3606 +torchvision/datasets/__pycache__/__init__.cpython-312.pyc,, +torchvision/datasets/__pycache__/_optical_flow.cpython-312.pyc,, +torchvision/datasets/__pycache__/_stereo_matching.cpython-312.pyc,, +torchvision/datasets/__pycache__/caltech.cpython-312.pyc,, +torchvision/datasets/__pycache__/celeba.cpython-312.pyc,, +torchvision/datasets/__pycache__/cifar.cpython-312.pyc,, +torchvision/datasets/__pycache__/cityscapes.cpython-312.pyc,, +torchvision/datasets/__pycache__/clevr.cpython-312.pyc,, +torchvision/datasets/__pycache__/coco.cpython-312.pyc,, +torchvision/datasets/__pycache__/country211.cpython-312.pyc,, +torchvision/datasets/__pycache__/dtd.cpython-312.pyc,, +torchvision/datasets/__pycache__/eurosat.cpython-312.pyc,, +torchvision/datasets/__pycache__/fakedata.cpython-312.pyc,, +torchvision/datasets/__pycache__/fer2013.cpython-312.pyc,, +torchvision/datasets/__pycache__/fgvc_aircraft.cpython-312.pyc,, +torchvision/datasets/__pycache__/flickr.cpython-312.pyc,, +torchvision/datasets/__pycache__/flowers102.cpython-312.pyc,, +torchvision/datasets/__pycache__/folder.cpython-312.pyc,, +torchvision/datasets/__pycache__/food101.cpython-312.pyc,, +torchvision/datasets/__pycache__/gtsrb.cpython-312.pyc,, +torchvision/datasets/__pycache__/hmdb51.cpython-312.pyc,, +torchvision/datasets/__pycache__/imagenet.cpython-312.pyc,, +torchvision/datasets/__pycache__/imagenette.cpython-312.pyc,, +torchvision/datasets/__pycache__/inaturalist.cpython-312.pyc,, +torchvision/datasets/__pycache__/kinetics.cpython-312.pyc,, +torchvision/datasets/__pycache__/kitti.cpython-312.pyc,, +torchvision/datasets/__pycache__/lfw.cpython-312.pyc,, +torchvision/datasets/__pycache__/lsun.cpython-312.pyc,, +torchvision/datasets/__pycache__/mnist.cpython-312.pyc,, +torchvision/datasets/__pycache__/moving_mnist.cpython-312.pyc,, +torchvision/datasets/__pycache__/omniglot.cpython-312.pyc,, +torchvision/datasets/__pycache__/oxford_iiit_pet.cpython-312.pyc,, +torchvision/datasets/__pycache__/pcam.cpython-312.pyc,, +torchvision/datasets/__pycache__/phototour.cpython-312.pyc,, +torchvision/datasets/__pycache__/places365.cpython-312.pyc,, +torchvision/datasets/__pycache__/rendered_sst2.cpython-312.pyc,, +torchvision/datasets/__pycache__/sbd.cpython-312.pyc,, +torchvision/datasets/__pycache__/sbu.cpython-312.pyc,, +torchvision/datasets/__pycache__/semeion.cpython-312.pyc,, +torchvision/datasets/__pycache__/stanford_cars.cpython-312.pyc,, +torchvision/datasets/__pycache__/stl10.cpython-312.pyc,, +torchvision/datasets/__pycache__/sun397.cpython-312.pyc,, +torchvision/datasets/__pycache__/svhn.cpython-312.pyc,, +torchvision/datasets/__pycache__/ucf101.cpython-312.pyc,, +torchvision/datasets/__pycache__/usps.cpython-312.pyc,, +torchvision/datasets/__pycache__/utils.cpython-312.pyc,, +torchvision/datasets/__pycache__/video_utils.cpython-312.pyc,, +torchvision/datasets/__pycache__/vision.cpython-312.pyc,, +torchvision/datasets/__pycache__/voc.cpython-312.pyc,, +torchvision/datasets/__pycache__/widerface.cpython-312.pyc,, +torchvision/datasets/_optical_flow.py,sha256=oRm_6rlBpJyi9d2IeTiebHssDEXQDKEKGw3ZqNVDMrg,19697 +torchvision/datasets/_stereo_matching.py,sha256=HzSLkWTXj26lpC9smajleUIgt_rsKEoull9NGW9jPE0,49058 +torchvision/datasets/caltech.py,sha256=IRpfSK1K-26f8TTQUojA9wWDHhXbPZRqZBIjJ_WhE60,8815 +torchvision/datasets/celeba.py,sha256=k-2W5IfCKpVTWb18LAHShnr_rwsMQ0CdqJ3REKSWJrk,8437 +torchvision/datasets/cifar.py,sha256=TqgqwpAMnr2LQ2iu6NdhkmYy8ChJ5YMpL2N_zTmqvdU,5791 +torchvision/datasets/cityscapes.py,sha256=a_FbxaDP2PRXM6TF44rK_vyCbziX7AxcZOYGCjmsO5o,10349 +torchvision/datasets/clevr.py,sha256=Yw2dTlep-ERTzIsKHPGL9cblF88mGlRcoGoBGac1XZ0,3460 +torchvision/datasets/coco.py,sha256=Zmfp6yZgWcDxXLDshcTnxDaKC6xvYsasPcBh_j9E9m4,4180 +torchvision/datasets/country211.py,sha256=T_WIsox6Ve6CxmFwnx6bX3KkLy1xzBCbAFBcGqHVYC8,2436 +torchvision/datasets/dtd.py,sha256=c6GtnNd4xj4BCE52GMaXnn-AnZm7yn9Yha8Iwb5xhCo,4019 +torchvision/datasets/eurosat.py,sha256=nKBDlYaYupwughReDD7Z_EH_WVTqqSyGRBjnIjmvUUk,2307 +torchvision/datasets/fakedata.py,sha256=gKmN6VyQzWjjeEPpLPxb9i4DWwW-MtGVJfZf8uwHgyo,2447 +torchvision/datasets/fer2013.py,sha256=f_Zj3Qf32x8ew5dZu8A03uph3I4AUvmmZabaLhTSMnU,5118 +torchvision/datasets/fgvc_aircraft.py,sha256=Y5P7SsYLeXDuxy7VHVTx9TYDKHloxtxlxT4JBDgbvXg,4626 +torchvision/datasets/flickr.py,sha256=rcbyRlYd-d_vRW9qmOPfX1bKBgFu4NbF-qlldqt2mcU,5431 +torchvision/datasets/flowers102.py,sha256=SdPXQtHAeZ5Iod0xyK2Xq7n0ENA6YIoEUFfRqiBu1Q0,4641 +torchvision/datasets/folder.py,sha256=bh7Jv0BOphBkKYxD-BogUWexE9RIrGR0FLM5MR24aGM,12919 +torchvision/datasets/food101.py,sha256=1vbbbahI-Lp9xySy5bsnS50TeV93ovesSIotY0astw0,3752 +torchvision/datasets/gtsrb.py,sha256=0n6GQIGPuKU7yA0tSpiAA1UktoShE2vzeA2EqhQZK-Q,3785 +torchvision/datasets/hmdb51.py,sha256=lC16QNHvbKkS8QfgVdhBvSwN2eLRFUBUNL021nkvgdc,5971 +torchvision/datasets/imagenet.py,sha256=kllmhLsUPgm88rww0j-OaEa-iuzGgyu49q6gphpXLjA,8691 +torchvision/datasets/imagenette.py,sha256=R7rW9UeDQYNhBA49FkzzP66689zwHm8hdvlv6P3_2b0,4267 +torchvision/datasets/inaturalist.py,sha256=izBRYXCGeCIRuhGP9BnwQpiJGNHFSneXqihPEDQwuas,9877 +torchvision/datasets/kinetics.py,sha256=q2NZZtS9AXiAh6HZazqAnqfWuFZepvDzfbwlhM2qhnw,9878 +torchvision/datasets/kitti.py,sha256=8mCScWNce0OdG3b6vWCJGR370CydbK2Iy8W96Dfsl0I,5637 +torchvision/datasets/lfw.py,sha256=C3bEBH7NmpqesMNqz1FGbq-KgT3mZwmr1aWBFqYBFpE,10501 +torchvision/datasets/lsun.py,sha256=m79MxqgRs04hLyj18tk5ZL1OFO4-Sc9ipd4gMlsqlnQ,5743 +torchvision/datasets/mnist.py,sha256=_XSFsJUxtSPzWb-hDb_A7CHRK1A285czOJTYpE2LP4s,21751 +torchvision/datasets/moving_mnist.py,sha256=6yCTZVgIlWy2f9bNlrAjpUWryeLohaWuN0bRhMdAERw,3644 +torchvision/datasets/omniglot.py,sha256=nu72y9QlMfufNKMxFaJMDz_vtP_0VZ3-1QrtR9xjfy0,4092 +torchvision/datasets/oxford_iiit_pet.py,sha256=t4me06AwjDjSTIE7f80VFuGxISGHFPz6B4Sn3uOrCBw,5519 +torchvision/datasets/pcam.py,sha256=Ub7UWrAufIzLXN8p6Cunt7osnHCNTL-sxDmEMGypq2Q,5285 +torchvision/datasets/phototour.py,sha256=V5UToh6KcI2hA5BlHwKVNdCXxb52YLTjS4lhz2JW8jQ,7868 +torchvision/datasets/places365.py,sha256=Qk92EEcwDrG1gBX3uVycakPZRQBb_ZOG-6tJz78qFXk,7070 +torchvision/datasets/rendered_sst2.py,sha256=2NRiL3I1hDrOdNllubdQ-gQ-Unaaqb2mLAXG4_JL5wY,3597 +torchvision/datasets/samplers/__init__.py,sha256=W1ZtQpGLG6aoHylo1t8PEsHIVoWwso5bSFk9JzKfH8g,161 +torchvision/datasets/samplers/__pycache__/__init__.cpython-312.pyc,, +torchvision/datasets/samplers/__pycache__/clip_sampler.cpython-312.pyc,, +torchvision/datasets/samplers/clip_sampler.py,sha256=1-k3bxToGpBlqC4-iyVDggtojA701NflW0nBRLK27tQ,6244 +torchvision/datasets/sbd.py,sha256=BpowMEO3_IxJgyjrtEN7XSLAKlrONVhCGr2kJXtTIzs,5414 +torchvision/datasets/sbu.py,sha256=QV_1aKa9xffZR4JZmyWg3izqoytfnuJKsYjcF4Mjqns,4084 +torchvision/datasets/semeion.py,sha256=-mIPORNUG5n7eNWq4VRLi9imlHm_bHsWKMv3E9HsAKo,3089 +torchvision/datasets/stanford_cars.py,sha256=UuNAZe6j7Z6g04RZEoYASPtpABwOI1--hAwxtcBxv18,4898 +torchvision/datasets/stl10.py,sha256=eQk04VL6mcKft57Yap7QATEXwerDWn5QrB-Lg7fVIX0,7234 +torchvision/datasets/sun397.py,sha256=q_qfa_rdx4GUklR9oIHCgQC0JUKXMc7UudTq6yUeJPQ,2783 +torchvision/datasets/svhn.py,sha256=Vk8VO74JUUaZHvejvkWJBRnmk-zpmHwjksMCZoBDWDc,4828 +torchvision/datasets/ucf101.py,sha256=s7rHl7qonY7PnmEZac_O2gmJUIVFzyNxVbvMY7IY_Io,5533 +torchvision/datasets/usps.py,sha256=7IP-xNZUJQNibubSodJgnpUJlCvNe-prd8BHsrbzSR0,3500 +torchvision/datasets/utils.py,sha256=uN8Tt0S3dAuGI-wjyMzlX59QSQbUJkfUwRf_J20uX4I,15994 +torchvision/datasets/video_utils.py,sha256=14GvzCRi7tbfeCq31MN9XP_6-bfewRSrvwavO4VBFdk,17213 +torchvision/datasets/vision.py,sha256=x8AuTqEBwwBoHmkkWD6Iki8o5LMxac2yhrzIFBDgodE,4249 +torchvision/datasets/voc.py,sha256=LhdQavn7-nq13zf9HIfjNYxPDa5SaTUDgayDe8uLfZc,8835 +torchvision/datasets/widerface.py,sha256=HVdxQcpuPYPzjT07lWFAsMFKEu9bNkUSrC4ihTok56o,8264 +torchvision/extension.py,sha256=YWBDURfCFXSmRvXi2iEg2L0hafN2-RnybpImh9JAUtQ,3141 +torchvision/image.so,sha256=SuYQDTx8zpTMDX9sC2lZ6UeX-GZwIFhxBJgUf-RRTDo,667257 +torchvision/io/__init__.py,sha256=-6ONNvXi97z0x3UjxLBwstOTG8KcJg_sFGha3oGFjAo,1583 +torchvision/io/__pycache__/__init__.cpython-312.pyc,, +torchvision/io/__pycache__/_load_gpu_decoder.cpython-312.pyc,, +torchvision/io/__pycache__/_video_opt.cpython-312.pyc,, +torchvision/io/__pycache__/image.cpython-312.pyc,, +torchvision/io/__pycache__/video.cpython-312.pyc,, +torchvision/io/__pycache__/video_reader.cpython-312.pyc,, +torchvision/io/_load_gpu_decoder.py,sha256=Cc8eP620qPDFc0q2qd-VYtjxtsgFPjOgg7Z04RXRziU,178 +torchvision/io/_video_opt.py,sha256=oW2Vvs13fa9nopb4Ot3n_VNiOUCn5ZPLQnH8Xf8-81g,20456 +torchvision/io/image.py,sha256=yXy4_g6_57LC-bEt3ukQbrpl6fsdeOaqDacGVJ6YtCc,21718 +torchvision/io/video.py,sha256=uU8TqYeeQ1TpODZFj0qyQrSCLByIzOP5di_3Pr9estQ,18025 +torchvision/io/video_reader.py,sha256=eI09x1vuUsbtL6rnyeiv894y8EA9bfdJakV1zWYzBtQ,11689 +torchvision/models/__init__.py,sha256=A8GQPE1bl3oUHpuD9ND53DV557IPY4459FNLW6sVXGI,865 +torchvision/models/__pycache__/__init__.cpython-312.pyc,, +torchvision/models/__pycache__/_api.cpython-312.pyc,, +torchvision/models/__pycache__/_meta.cpython-312.pyc,, +torchvision/models/__pycache__/_utils.cpython-312.pyc,, +torchvision/models/__pycache__/alexnet.cpython-312.pyc,, +torchvision/models/__pycache__/convnext.cpython-312.pyc,, +torchvision/models/__pycache__/densenet.cpython-312.pyc,, +torchvision/models/__pycache__/efficientnet.cpython-312.pyc,, +torchvision/models/__pycache__/feature_extraction.cpython-312.pyc,, +torchvision/models/__pycache__/googlenet.cpython-312.pyc,, +torchvision/models/__pycache__/inception.cpython-312.pyc,, +torchvision/models/__pycache__/maxvit.cpython-312.pyc,, +torchvision/models/__pycache__/mnasnet.cpython-312.pyc,, +torchvision/models/__pycache__/mobilenet.cpython-312.pyc,, +torchvision/models/__pycache__/mobilenetv2.cpython-312.pyc,, +torchvision/models/__pycache__/mobilenetv3.cpython-312.pyc,, +torchvision/models/__pycache__/regnet.cpython-312.pyc,, +torchvision/models/__pycache__/resnet.cpython-312.pyc,, +torchvision/models/__pycache__/shufflenetv2.cpython-312.pyc,, +torchvision/models/__pycache__/squeezenet.cpython-312.pyc,, +torchvision/models/__pycache__/swin_transformer.cpython-312.pyc,, +torchvision/models/__pycache__/vgg.cpython-312.pyc,, +torchvision/models/__pycache__/vision_transformer.cpython-312.pyc,, +torchvision/models/_api.py,sha256=uIIJnxX1zYMNpdvJ0haSq15_XlR1QteFZBYVAdtEheg,10054 +torchvision/models/_meta.py,sha256=fqpeQBsf9EEYbmApQ8Q0LKyM9_UFwjireII5mwDbwJY,28875 +torchvision/models/_utils.py,sha256=S8uDD7maNefy-fEW6mpz8dFU68acK1HxN0kt1qpkkDo,10893 +torchvision/models/alexnet.py,sha256=dvBZLVH60TOTHCNNkWg0TFLtuJ5Ghh_xXN73r3Vyq58,4488 +torchvision/models/convnext.py,sha256=tP73tH-us6h2KSdVcPypEX9Izk5lsr82KsGT15mj4NE,15326 +torchvision/models/densenet.py,sha256=OZEsHJw76kOSRG4TKhLy7lPGsGEixy6llHkpC8snSOo,16825 +torchvision/models/detection/__init__.py,sha256=JwYm_fTGO_FeRg4eTOQLwQPZ9lC9jheZ-QEoJgqKTjg,168 +torchvision/models/detection/__pycache__/__init__.cpython-312.pyc,, +torchvision/models/detection/__pycache__/_utils.cpython-312.pyc,, +torchvision/models/detection/__pycache__/anchor_utils.cpython-312.pyc,, +torchvision/models/detection/__pycache__/backbone_utils.cpython-312.pyc,, +torchvision/models/detection/__pycache__/faster_rcnn.cpython-312.pyc,, +torchvision/models/detection/__pycache__/fcos.cpython-312.pyc,, +torchvision/models/detection/__pycache__/generalized_rcnn.cpython-312.pyc,, +torchvision/models/detection/__pycache__/image_list.cpython-312.pyc,, +torchvision/models/detection/__pycache__/keypoint_rcnn.cpython-312.pyc,, +torchvision/models/detection/__pycache__/mask_rcnn.cpython-312.pyc,, +torchvision/models/detection/__pycache__/retinanet.cpython-312.pyc,, +torchvision/models/detection/__pycache__/roi_heads.cpython-312.pyc,, +torchvision/models/detection/__pycache__/rpn.cpython-312.pyc,, +torchvision/models/detection/__pycache__/ssd.cpython-312.pyc,, +torchvision/models/detection/__pycache__/ssdlite.cpython-312.pyc,, +torchvision/models/detection/__pycache__/transform.cpython-312.pyc,, +torchvision/models/detection/_utils.py,sha256=2y3FQ4F5yXhFM7VIWmu_70FpKgZjxdT_ucfzYwi3ZUQ,22127 +torchvision/models/detection/anchor_utils.py,sha256=8Ix1Vp3i2kgJGr6esie3rw0_yAjtrUSvLXVKPaoZeQo,11859 +torchvision/models/detection/backbone_utils.py,sha256=4FyzocR6YS7cG5IJTMRwC44tupbXQDA_Ru_8qqaju2I,10548 +torchvision/models/detection/faster_rcnn.py,sha256=8DnegLKZnr8Q-zrzGT7_peIc_k_R1q1ijDH5n1P3gQE,36979 +torchvision/models/detection/fcos.py,sha256=8r8MayvUMeTKfDoza4Hy67ChgRglLzBG6YS5qNe84sM,34235 +torchvision/models/detection/generalized_rcnn.py,sha256=4-Dp8Vx-SjDDSZ7TsZ11rmkvEH336aLuSOlERXiQ7fs,4743 +torchvision/models/detection/image_list.py,sha256=SUJ3xMn-1xc6ivYZUNIdWBh3RH9xD8EtCdpsXnPI_iM,783 +torchvision/models/detection/keypoint_rcnn.py,sha256=4HxwRrp8lJfdyi8K3eBq4vstbRrL8bZc2Hhh-pVHjsI,21947 +torchvision/models/detection/mask_rcnn.py,sha256=X1GQS314qOy4uCCp7MPfH6W12IydRwW-tDCmCnB1FGg,26713 +torchvision/models/detection/retinanet.py,sha256=17Q0RdqqugASEVDGJfr8lCD61zjEqD5XxwQZAmZUZ24,37300 +torchvision/models/detection/roi_heads.py,sha256=Uh9950xZUEmejwD2pRRhKvqNV0bY_G2Om8yGC2EdDDg,33822 +torchvision/models/detection/rpn.py,sha256=7jbqPpLelnGCb5Fn-muUXeZF9EQ2nhE5r2aNAuR9V0M,15838 +torchvision/models/detection/ssd.py,sha256=tbsgVbRD36WrjkZEB4xi1fvOXT62ry0p8G_Sd-j5CrY,28979 +torchvision/models/detection/ssdlite.py,sha256=8nyEUYONUYe319JpgevKEfjr_FxCgDNU8gOyfuZ3L3c,13219 +torchvision/models/detection/transform.py,sha256=Ma0CDvLCMlk3MxS3asXcDxrSosRLacaLpi-T34LXm1A,12189 +torchvision/models/efficientnet.py,sha256=4qyeoXkYGFyUsBDt8TygDYycMMt1zhGwB_l4PmoPv4g,43090 +torchvision/models/feature_extraction.py,sha256=8zZi6BF-3grrGAC61mgYzzvoYk_tSEZOWojBUK7FRBM,27933 +torchvision/models/googlenet.py,sha256=b_TaeoABP08AVa8VqTreDdAuAnQTl54b6B1JOHbLy3A,12806 +torchvision/models/inception.py,sha256=ifrLErzOVG-vlwQOMXLX5yMgcpHxCQQ17L7Wacn5QhQ,18851 +torchvision/models/maxvit.py,sha256=AZkORRMQt7QRlPP-AiUxdwxGUEV6mR408KYbwUuxTSE,32096 +torchvision/models/mnasnet.py,sha256=h9jY1TupaChZj9khnXya_l4O1exUWhWOOCmhJCCImKc,17574 +torchvision/models/mobilenet.py,sha256=lSRVxw2TL3LFBwCadvyvH6n3GzqUTnK2-rhX3MOgSrs,211 +torchvision/models/mobilenetv2.py,sha256=v9cRBAp7_C_50JFkjGZ0luvuh45oCYgYn37pcG2UL8o,9710 +torchvision/models/mobilenetv3.py,sha256=-Xk32m_Wdn-ap8wCL4Tl7wjiROIwDwhasInYTMwwOrE,16279 +torchvision/models/optical_flow/__init__.py,sha256=0zRlMWQJCjFqoUafUXVgO89-z7em7tACo9E8hHSq9RQ,20 +torchvision/models/optical_flow/__pycache__/__init__.cpython-312.pyc,, +torchvision/models/optical_flow/__pycache__/_utils.cpython-312.pyc,, +torchvision/models/optical_flow/__pycache__/raft.cpython-312.pyc,, +torchvision/models/optical_flow/_utils.py,sha256=v-tQJzYmYukrD1sQAE-5j5jxyvComwF1UdGkz5tVTLw,2077 +torchvision/models/optical_flow/raft.py,sha256=FpSLPXisugu5Rzp_D5XCr037snBapMJ0dDPrw9c3CNk,39995 +torchvision/models/quantization/__init__.py,sha256=gqFM7zI4UUHKKBDJAumozOn7xPL0JtvyNS8Ejz6QXp0,125 +torchvision/models/quantization/__pycache__/__init__.cpython-312.pyc,, +torchvision/models/quantization/__pycache__/googlenet.cpython-312.pyc,, +torchvision/models/quantization/__pycache__/inception.cpython-312.pyc,, +torchvision/models/quantization/__pycache__/mobilenet.cpython-312.pyc,, +torchvision/models/quantization/__pycache__/mobilenetv2.cpython-312.pyc,, +torchvision/models/quantization/__pycache__/mobilenetv3.cpython-312.pyc,, +torchvision/models/quantization/__pycache__/resnet.cpython-312.pyc,, +torchvision/models/quantization/__pycache__/shufflenetv2.cpython-312.pyc,, +torchvision/models/quantization/__pycache__/utils.cpython-312.pyc,, +torchvision/models/quantization/googlenet.py,sha256=C-8lm9TnjkEuwu6zaPp0r5mb0QMYvTMGOtz2--s1IFo,8080 +torchvision/models/quantization/inception.py,sha256=hg8K1QNk7T-Qo3zOB47eupS3Thu_RjVI6mG2HzAEx8M,10815 +torchvision/models/quantization/mobilenet.py,sha256=lSRVxw2TL3LFBwCadvyvH6n3GzqUTnK2-rhX3MOgSrs,211 +torchvision/models/quantization/mobilenetv2.py,sha256=ggpNLU4_JkyMn8IPTgj1p0xx_Wvspcii2Wd3ISj5tBE,5883 +torchvision/models/quantization/mobilenetv3.py,sha256=PVWmSP62Pn8hQkd682l6uYFLQp1nxZltMOE-FhhO9OU,9230 +torchvision/models/quantization/resnet.py,sha256=9Hb6KyPv33Jj1A6JciXvGX06q0RkwwP10u8GxFfmorM,17939 +torchvision/models/quantization/shufflenetv2.py,sha256=eS2y34ZTG03dNJgtVJ2qSXQWZ22PHIWBYeC8cbvI1yI,16884 +torchvision/models/quantization/utils.py,sha256=n8mWsK9_Ek_M2AqGKPLoLlcKaYGH2PrF2l5_W84oBMk,2058 +torchvision/models/regnet.py,sha256=-7s5n0qzXZPR9HgzOk9aj1sv9dWZ3AxnP7CmZRdUeZI,63553 +torchvision/models/resnet.py,sha256=dJmlBZrXsaH491Q8BLShN5UUD62DfDhTC0j_XZYQv24,38932 +torchvision/models/segmentation/__init__.py,sha256=TGk6UdVXAMtwBpYalrvdXZnmSwqzTDOT1lgKrfzhHrQ,66 +torchvision/models/segmentation/__pycache__/__init__.cpython-312.pyc,, +torchvision/models/segmentation/__pycache__/_utils.cpython-312.pyc,, +torchvision/models/segmentation/__pycache__/deeplabv3.cpython-312.pyc,, +torchvision/models/segmentation/__pycache__/fcn.cpython-312.pyc,, +torchvision/models/segmentation/__pycache__/lraspp.cpython-312.pyc,, +torchvision/models/segmentation/_utils.py,sha256=QfyqCtH_MJnIkKW5m-98GZD2MjtPYLtPTDi79pcIGhs,1197 +torchvision/models/segmentation/deeplabv3.py,sha256=wVgXz21sugSck2KbG7WD-wgMwCAW0wd8jBGhgue300s,15015 +torchvision/models/segmentation/fcn.py,sha256=I1FqaZZVPc3Fbg_7E2L5qpumnupxBYc7KYsW03EG_Cs,8973 +torchvision/models/segmentation/lraspp.py,sha256=dt5DJ_qbDZlEM0SIuN87JU43JHfVlb8Oepp76KDv5tw,7643 +torchvision/models/shufflenetv2.py,sha256=84FiPfkhJpSw6Q9Jmaug5MW5qmWCO3VhAPF61EiMn7Q,15444 +torchvision/models/squeezenet.py,sha256=apjFPEI5nr_493bAQsR245EorzaMYXVQSqdcveyAfy0,8763 +torchvision/models/swin_transformer.py,sha256=VwvnImWcjblashj0OONycDJnIkz-zRDpm365v_a0-zo,39337 +torchvision/models/vgg.py,sha256=jYjIoY2jtKAc-aURCQsvbgBxup1Gh4fVZSt2NzFLlZY,19225 +torchvision/models/video/__init__.py,sha256=O4HB-RaXgCtnvpMDAuMBaIeKIiYEkNxra_fmAHLUIJM,93 +torchvision/models/video/__pycache__/__init__.cpython-312.pyc,, +torchvision/models/video/__pycache__/mvit.cpython-312.pyc,, +torchvision/models/video/__pycache__/resnet.cpython-312.pyc,, +torchvision/models/video/__pycache__/s3d.cpython-312.pyc,, +torchvision/models/video/__pycache__/swin_transformer.cpython-312.pyc,, +torchvision/models/video/mvit.py,sha256=0AZ31K5QcUBWZUUPTI1FCCM2Fma95bPs1o82zzpw2i0,32998 +torchvision/models/video/resnet.py,sha256=RUnbUXFmoWNo_XbEKLmVSM8LUDcyv6jGZJ8GGpZi_6U,16771 +torchvision/models/video/s3d.py,sha256=jx9gMP18Bzb7UO3vjejVBHlrCrJPdWFDfTn7XeU5kMg,7815 +torchvision/models/video/swin_transformer.py,sha256=3GMyPGPeMcwJ1p9TGiRbpIlP-G7Qv_jWNbZmqIwMNyA,27688 +torchvision/models/vision_transformer.py,sha256=O4mdBjYFsp-HTZA9bXfux_wJzIPRv1uS43PjuNh52zc,32136 +torchvision/ops/__init__.py,sha256=eVv16QSBwgKaojOUHMPCy4ou9ZeFh-HoCV4DpqrZG4U,1928 +torchvision/ops/__pycache__/__init__.cpython-312.pyc,, +torchvision/ops/__pycache__/_box_convert.cpython-312.pyc,, +torchvision/ops/__pycache__/_register_onnx_ops.cpython-312.pyc,, +torchvision/ops/__pycache__/_utils.cpython-312.pyc,, +torchvision/ops/__pycache__/boxes.cpython-312.pyc,, +torchvision/ops/__pycache__/ciou_loss.cpython-312.pyc,, +torchvision/ops/__pycache__/deform_conv.cpython-312.pyc,, +torchvision/ops/__pycache__/diou_loss.cpython-312.pyc,, +torchvision/ops/__pycache__/drop_block.cpython-312.pyc,, +torchvision/ops/__pycache__/feature_pyramid_network.cpython-312.pyc,, +torchvision/ops/__pycache__/focal_loss.cpython-312.pyc,, +torchvision/ops/__pycache__/giou_loss.cpython-312.pyc,, +torchvision/ops/__pycache__/misc.cpython-312.pyc,, +torchvision/ops/__pycache__/poolers.cpython-312.pyc,, +torchvision/ops/__pycache__/ps_roi_align.cpython-312.pyc,, +torchvision/ops/__pycache__/ps_roi_pool.cpython-312.pyc,, +torchvision/ops/__pycache__/roi_align.cpython-312.pyc,, +torchvision/ops/__pycache__/roi_pool.cpython-312.pyc,, +torchvision/ops/__pycache__/stochastic_depth.cpython-312.pyc,, +torchvision/ops/_box_convert.py,sha256=_bRRpErwk03rcPuscO1tCI9v3l88oNlDBDl2jzPlbKo,2409 +torchvision/ops/_register_onnx_ops.py,sha256=Fyb1kC2m2OqZdfW_M86pt9-S66e1qNUhXNu1EQRa034,4181 +torchvision/ops/_utils.py,sha256=pVHPpsmx6XcfGjUVk-XAEnd8QJBkrw_cT6fO_IwICE4,3630 +torchvision/ops/boxes.py,sha256=ti47P4NwxY3OPRowTVBinsXFWVsqYPMrHOEsdJCwP5E,16605 +torchvision/ops/ciou_loss.py,sha256=3HClrMMKOJ3bndIUinNp3cp6Cim4-ZmmfuLn1-NPDUo,2756 +torchvision/ops/deform_conv.py,sha256=fJxkVR_p_OQMzMja4flvmTgqDPvrOOcwzDG8bV7Q7pE,6990 +torchvision/ops/diou_loss.py,sha256=tssNJhII4WT-wmidFS8gFNteQIAJz-Nd1Q7Trz1BjIY,3362 +torchvision/ops/drop_block.py,sha256=A4EGIl7txrU_QmkI1N0W9hfd8tq8yx6zq32oYXaddLQ,5855 +torchvision/ops/feature_pyramid_network.py,sha256=mfkaygxRz-0TAdTMq2fCAL-E0WxlRnTfdb-s_J5qPE4,8702 +torchvision/ops/focal_loss.py,sha256=9kFqGyA0-hodRw9Au74k-FuS14OhsAvbFxDGvpx08Sg,2261 +torchvision/ops/giou_loss.py,sha256=OXSaMZDZ0qy7jgaQ9exB_DMQXzcATBAFiIjzSlOV-bQ,2696 +torchvision/ops/misc.py,sha256=yFnK7GT9OCMfDrn4NtQXKdh5broi1xocL94SoyqhWuw,13572 +torchvision/ops/poolers.py,sha256=zzYhH7poMwGlYxDvAvCaL9emg9X7sM4xZFLEy0zvv5s,11920 +torchvision/ops/ps_roi_align.py,sha256=4iAbeUVTessAcxvJhuARN_aFGUTZC9R4KrKC_mBH3MQ,3625 +torchvision/ops/ps_roi_pool.py,sha256=jOv-2pAZdLFvvt4r4NwiRfxU5WAOy_vi6gxZjMvlusw,2870 +torchvision/ops/roi_align.py,sha256=Ig9jLul90wBM3kaZuYEutsJEXfaCo3D0s_PxYMr9jQc,11292 +torchvision/ops/roi_pool.py,sha256=70ou6Xc7qJxKe3SC54QIW3L99PoS0gLlwGocaYDbD2w,2943 +torchvision/ops/stochastic_depth.py,sha256=ISZ9noJyZLxpTG-wa2VmPs66qjhVsP7ZxWHvumWSP3U,2236 +torchvision/transforms/__init__.py,sha256=EMft42B1JAiU11J1rxIN4Znis6EJPbp-bsGjAzH-24M,53 +torchvision/transforms/__pycache__/__init__.cpython-312.pyc,, +torchvision/transforms/__pycache__/_functional_pil.cpython-312.pyc,, +torchvision/transforms/__pycache__/_functional_tensor.cpython-312.pyc,, +torchvision/transforms/__pycache__/_functional_video.cpython-312.pyc,, +torchvision/transforms/__pycache__/_presets.cpython-312.pyc,, +torchvision/transforms/__pycache__/_transforms_video.cpython-312.pyc,, +torchvision/transforms/__pycache__/autoaugment.cpython-312.pyc,, +torchvision/transforms/__pycache__/functional.cpython-312.pyc,, +torchvision/transforms/__pycache__/transforms.cpython-312.pyc,, +torchvision/transforms/_functional_pil.py,sha256=LZO3ZBJRECXQJCDK9h_1TDYOFOajN_QSeASLQr1xMPg,12070 +torchvision/transforms/_functional_tensor.py,sha256=3cEs8IYfRNQyff5Iriv--cZTWOIfvw2eaWiHU1-94AE,33939 +torchvision/transforms/_functional_video.py,sha256=YcV557YglbJsq9SRGJHFoRbtxawiLSJ1oM5rV75OyqQ,3857 +torchvision/transforms/_presets.py,sha256=RAjD6DgpU4QnNxV0MfZ3uHgzuARf-cdxD3Vo_KKIYeY,8510 +torchvision/transforms/_transforms_video.py,sha256=Buz5LCWVPGiEonHE-cXIXfbkBhNc0qxVraxkNdxKp8o,4950 +torchvision/transforms/autoaugment.py,sha256=JcbdEDbR0-OqTE4cwkhVSB45woFZQ_Fq5xmjFu_3bjg,28243 +torchvision/transforms/functional.py,sha256=r9DojEr-0oqCOLuSMH0B4kWtI3UEbY_4jS7RBWDZKqM,67855 +torchvision/transforms/transforms.py,sha256=eRIUr0I1i7BMqrXm4xsBJQYYGpvIkDr_VMsctQOI0M4,85547 +torchvision/transforms/v2/__init__.py,sha256=UUJgzZguNPl7B33Zt3gexO7gSApSuWHTpzE7fNXQpA0,1545 +torchvision/transforms/v2/__pycache__/__init__.cpython-312.pyc,, +torchvision/transforms/v2/__pycache__/_augment.cpython-312.pyc,, +torchvision/transforms/v2/__pycache__/_auto_augment.cpython-312.pyc,, +torchvision/transforms/v2/__pycache__/_color.cpython-312.pyc,, +torchvision/transforms/v2/__pycache__/_container.cpython-312.pyc,, +torchvision/transforms/v2/__pycache__/_deprecated.cpython-312.pyc,, +torchvision/transforms/v2/__pycache__/_geometry.cpython-312.pyc,, +torchvision/transforms/v2/__pycache__/_meta.cpython-312.pyc,, +torchvision/transforms/v2/__pycache__/_misc.cpython-312.pyc,, +torchvision/transforms/v2/__pycache__/_temporal.cpython-312.pyc,, +torchvision/transforms/v2/__pycache__/_transform.cpython-312.pyc,, +torchvision/transforms/v2/__pycache__/_type_conversion.cpython-312.pyc,, +torchvision/transforms/v2/__pycache__/_utils.cpython-312.pyc,, +torchvision/transforms/v2/_augment.py,sha256=YnxyCNZvYYredkImS-IasQ_HkBakHs0y2cXkdfSg2ek,16154 +torchvision/transforms/v2/_auto_augment.py,sha256=sQWkEF1N17XU4F6nBGva7kUuiuGNEOCAGHYGn8oa0A8,32025 +torchvision/transforms/v2/_color.py,sha256=cYulJPZkwv6FpBBxwSe4Lq3qXltf6vos5urxvBR4OL4,16978 +torchvision/transforms/v2/_container.py,sha256=SFh-FU8ceir934hxS_VkbVQq0SxzGSULPaYpouJJhPs,6055 +torchvision/transforms/v2/_deprecated.py,sha256=X_80zQwRKzGEcs8yq9JqhKBK1W56J121kyaQv_PVkQM,1946 +torchvision/transforms/v2/_geometry.py,sha256=xjd2_TavgN8Dm_xphLmRcIvwnpcu1KLz3nPzZpUoQYM,67585 +torchvision/transforms/v2/_meta.py,sha256=mTunFNfyvhG3arMGnUJH6Cy5MRlrHOIfkv4SxlAhWaI,1403 +torchvision/transforms/v2/_misc.py,sha256=e3M60ihEhkx4Riuj5DOIhWhWNo-QLSI-pmreJhJiSRM,19102 +torchvision/transforms/v2/_temporal.py,sha256=1sd0pJuou0l9gNHjHFA40dSYRB-JkDvvFOia7WcIQ50,905 +torchvision/transforms/v2/_transform.py,sha256=QP35AB088j2Z6ZUwGAI6S5mM2FsXtX_5nKuKp1orA2s,9367 +torchvision/transforms/v2/_type_conversion.py,sha256=DJLueqE-07JADokRTY3fD9_qktdRpFDzGn9dxzXCT1A,2856 +torchvision/transforms/v2/_utils.py,sha256=zOOoaiOfivVfIVTo5GLSAAlknhcr6G7A9Tnkclrl0OQ,8846 +torchvision/transforms/v2/functional/__init__.py,sha256=4SDjzgj9e4oM4IUKy9YJAwXFnBoLpygd8sSM_7sMvK0,3546 +torchvision/transforms/v2/functional/__pycache__/__init__.cpython-312.pyc,, +torchvision/transforms/v2/functional/__pycache__/_augment.cpython-312.pyc,, +torchvision/transforms/v2/functional/__pycache__/_color.cpython-312.pyc,, +torchvision/transforms/v2/functional/__pycache__/_deprecated.cpython-312.pyc,, +torchvision/transforms/v2/functional/__pycache__/_geometry.cpython-312.pyc,, +torchvision/transforms/v2/functional/__pycache__/_meta.cpython-312.pyc,, +torchvision/transforms/v2/functional/__pycache__/_misc.cpython-312.pyc,, +torchvision/transforms/v2/functional/__pycache__/_temporal.cpython-312.pyc,, +torchvision/transforms/v2/functional/__pycache__/_type_conversion.cpython-312.pyc,, +torchvision/transforms/v2/functional/__pycache__/_utils.cpython-312.pyc,, +torchvision/transforms/v2/functional/_augment.py,sha256=MRM8E3_gKfTTC0qFt3cKI4UxTxQtuGI9MeY2mBsrj04,3473 +torchvision/transforms/v2/functional/_color.py,sha256=nUASg1bTHmsf2AT_1Q7CLNXhObrRPbB1w2fDuz9k5e8,30244 +torchvision/transforms/v2/functional/_deprecated.py,sha256=ycYZLDwDyd612aPbTKIV3gqhCRLMdF03MQELct4LeGs,801 +torchvision/transforms/v2/functional/_geometry.py,sha256=5QL4IdQV72PkJX61c4A5M4WLq60ihTQB6g1PE9tMqmM,87520 +torchvision/transforms/v2/functional/_meta.py,sha256=AxTEF6mdybAW1lC_DcjfKlxvSuiVupnqbJJrqS5x4lc,10547 +torchvision/transforms/v2/functional/_misc.py,sha256=OXu4GTCF9i_1lz7T62gKcEs94faBO7wyYmpUOCnkUEY,17517 +torchvision/transforms/v2/functional/_temporal.py,sha256=24CQCXXO12TnW7aUiUQdrk5DRSpTPONjjC4jaGh3lH4,1136 +torchvision/transforms/v2/functional/_type_conversion.py,sha256=78wl0dNPwX08jOCW6KcZSGy8RAQqyxMtdrTUQVQlUTM,869 +torchvision/transforms/v2/functional/_utils.py,sha256=tsmwIF37Z9QnP9x3x4hAs1hLrcvL78GLkuO6Rq1EUTk,5479 +torchvision/tv_tensors/__init__.py,sha256=C6N8p5aulpehsOBBmH1cPIY1xiOSASZVBfnlXgGvR_s,1509 +torchvision/tv_tensors/__pycache__/__init__.cpython-312.pyc,, +torchvision/tv_tensors/__pycache__/_bounding_boxes.cpython-312.pyc,, +torchvision/tv_tensors/__pycache__/_dataset_wrapper.cpython-312.pyc,, +torchvision/tv_tensors/__pycache__/_image.cpython-312.pyc,, +torchvision/tv_tensors/__pycache__/_mask.cpython-312.pyc,, +torchvision/tv_tensors/__pycache__/_torch_function_helpers.cpython-312.pyc,, +torchvision/tv_tensors/__pycache__/_tv_tensor.cpython-312.pyc,, +torchvision/tv_tensors/__pycache__/_video.cpython-312.pyc,, +torchvision/tv_tensors/_bounding_boxes.py,sha256=_-bDwN1gnHpfnHXEK0O6bQrcEOv656VOliHOgoNstpw,4493 +torchvision/tv_tensors/_dataset_wrapper.py,sha256=fNnk3CSXipBNFsmnsPpa10DRN0I_Ly4Xib2Y5Zng9Ro,24505 +torchvision/tv_tensors/_image.py,sha256=bwx4n8qObrknE3xEIDJOs0vWJzCg4XISjtXR7ksJTgs,1934 +torchvision/tv_tensors/_mask.py,sha256=s85DdYFK6cyrL0_MnhAC2jTJxZzL7MJ8DTx985JPVhQ,1478 +torchvision/tv_tensors/_torch_function_helpers.py,sha256=81qDZqgzUeSgfSeWhsrw1Ukwltvf97WbwmKWHm7X8X0,2276 +torchvision/tv_tensors/_tv_tensor.py,sha256=dGQJhvOVTjb1LVT5qPZLJxox30uDMmODB26Iz6TjVbc,6248 +torchvision/tv_tensors/_video.py,sha256=4dQ5Rh_0ghPtaLVSOxVWXJv1uWi8ZKXlfbRsBZ3roxw,1416 +torchvision/utils.py,sha256=Bu04d39mBjf2Q_eMb_uZVtLL5tGl3HhC18VvB_lXgQM,27306 +torchvision/version.py,sha256=ZU-VOJy0kSuBVeSZSUN2GnuMwkjzmN-nOOa4Y54S-0M,203 diff --git a/lib/python3.12/site-packages/torchvision-0.21.0.dist-info/REQUESTED b/lib/python3.12/site-packages/torchvision-0.21.0.dist-info/REQUESTED new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/torchvision-0.21.0.dist-info/WHEEL b/lib/python3.12/site-packages/torchvision-0.21.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..01e196cb28ea1f2113c136f125d1bf35b2affbbf --- /dev/null +++ b/lib/python3.12/site-packages/torchvision-0.21.0.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: setuptools (72.1.0) +Root-Is-Purelib: false +Tag: cp312-cp312-linux_x86_64 + diff --git a/lib/python3.12/site-packages/torchvision-0.21.0.dist-info/top_level.txt b/lib/python3.12/site-packages/torchvision-0.21.0.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..e35531e566f2a925d851b9d3b8fa99645838e6e0 --- /dev/null +++ b/lib/python3.12/site-packages/torchvision-0.21.0.dist-info/top_level.txt @@ -0,0 +1 @@ +torchvision