ZhengyangZhang commited on
Commit
2815277
·
verified ·
1 Parent(s): a102c6e

Add files using upload-large-folder tool

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/INSTALLER +1 -0
  2. lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/METADATA +112 -0
  3. lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/RECORD +9 -0
  4. lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/WHEEL +5 -0
  5. lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/licenses/LICENSE +201 -0
  6. lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/top_level.txt +1 -0
  7. lib/python3.12/site-packages/datasets/__pycache__/__init__.cpython-312.pyc +0 -0
  8. lib/python3.12/site-packages/datasets/__pycache__/arrow_reader.cpython-312.pyc +0 -0
  9. lib/python3.12/site-packages/datasets/__pycache__/arrow_writer.cpython-312.pyc +0 -0
  10. lib/python3.12/site-packages/datasets/__pycache__/builder.cpython-312.pyc +0 -0
  11. lib/python3.12/site-packages/datasets/__pycache__/combine.cpython-312.pyc +0 -0
  12. lib/python3.12/site-packages/datasets/__pycache__/config.cpython-312.pyc +0 -0
  13. lib/python3.12/site-packages/datasets/__pycache__/data_files.cpython-312.pyc +0 -0
  14. lib/python3.12/site-packages/datasets/__pycache__/distributed.cpython-312.pyc +0 -0
  15. lib/python3.12/site-packages/datasets/__pycache__/exceptions.cpython-312.pyc +0 -0
  16. lib/python3.12/site-packages/datasets/__pycache__/fingerprint.cpython-312.pyc +0 -0
  17. lib/python3.12/site-packages/datasets/__pycache__/hub.cpython-312.pyc +0 -0
  18. lib/python3.12/site-packages/datasets/__pycache__/info.cpython-312.pyc +0 -0
  19. lib/python3.12/site-packages/datasets/__pycache__/inspect.cpython-312.pyc +0 -0
  20. lib/python3.12/site-packages/datasets/__pycache__/keyhash.cpython-312.pyc +0 -0
  21. lib/python3.12/site-packages/datasets/__pycache__/load.cpython-312.pyc +0 -0
  22. lib/python3.12/site-packages/datasets/__pycache__/naming.cpython-312.pyc +0 -0
  23. lib/python3.12/site-packages/datasets/__pycache__/search.cpython-312.pyc +0 -0
  24. lib/python3.12/site-packages/datasets/__pycache__/splits.cpython-312.pyc +0 -0
  25. lib/python3.12/site-packages/datasets/__pycache__/streaming.cpython-312.pyc +0 -0
  26. lib/python3.12/site-packages/datasets/download/__init__.py +10 -0
  27. lib/python3.12/site-packages/datasets/download/__pycache__/__init__.cpython-312.pyc +0 -0
  28. lib/python3.12/site-packages/datasets/download/__pycache__/download_config.cpython-312.pyc +0 -0
  29. lib/python3.12/site-packages/datasets/download/__pycache__/download_manager.cpython-312.pyc +0 -0
  30. lib/python3.12/site-packages/datasets/download/__pycache__/streaming_download_manager.cpython-312.pyc +0 -0
  31. lib/python3.12/site-packages/datasets/download/download_config.py +81 -0
  32. lib/python3.12/site-packages/datasets/download/download_manager.py +340 -0
  33. lib/python3.12/site-packages/datasets/download/streaming_download_manager.py +219 -0
  34. lib/python3.12/site-packages/datasets/features/__init__.py +23 -0
  35. lib/python3.12/site-packages/datasets/features/__pycache__/__init__.cpython-312.pyc +0 -0
  36. lib/python3.12/site-packages/datasets/features/__pycache__/audio.cpython-312.pyc +0 -0
  37. lib/python3.12/site-packages/datasets/features/__pycache__/image.cpython-312.pyc +0 -0
  38. lib/python3.12/site-packages/datasets/features/__pycache__/pdf.cpython-312.pyc +0 -0
  39. lib/python3.12/site-packages/datasets/features/__pycache__/translation.cpython-312.pyc +0 -0
  40. lib/python3.12/site-packages/datasets/features/__pycache__/video.cpython-312.pyc +0 -0
  41. lib/python3.12/site-packages/datasets/features/audio.py +274 -0
  42. lib/python3.12/site-packages/datasets/features/features.py +2332 -0
  43. lib/python3.12/site-packages/datasets/features/image.py +382 -0
  44. lib/python3.12/site-packages/datasets/features/pdf.py +238 -0
  45. lib/python3.12/site-packages/datasets/features/translation.py +129 -0
  46. lib/python3.12/site-packages/datasets/features/video.py +280 -0
  47. lib/python3.12/site-packages/datasets/filesystems/__init__.py +47 -0
  48. lib/python3.12/site-packages/datasets/filesystems/__pycache__/__init__.cpython-312.pyc +0 -0
  49. lib/python3.12/site-packages/datasets/filesystems/__pycache__/compression.cpython-312.pyc +0 -0
  50. lib/python3.12/site-packages/datasets/filesystems/compression.py +127 -0
lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/INSTALLER ADDED
@@ -0,0 +1 @@
 
 
1
+ pip
lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/METADATA ADDED
@@ -0,0 +1,112 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Metadata-Version: 2.4
2
+ Name: aiosignal
3
+ Version: 1.4.0
4
+ Summary: aiosignal: a list of registered asynchronous callbacks
5
+ Home-page: https://github.com/aio-libs/aiosignal
6
+ Maintainer: aiohttp team <team@aiohttp.org>
7
+ Maintainer-email: team@aiohttp.org
8
+ License: Apache 2.0
9
+ Project-URL: Chat: Gitter, https://gitter.im/aio-libs/Lobby
10
+ Project-URL: CI: GitHub Actions, https://github.com/aio-libs/aiosignal/actions
11
+ Project-URL: Coverage: codecov, https://codecov.io/github/aio-libs/aiosignal
12
+ Project-URL: Docs: RTD, https://docs.aiosignal.org
13
+ Project-URL: GitHub: issues, https://github.com/aio-libs/aiosignal/issues
14
+ Project-URL: GitHub: repo, https://github.com/aio-libs/aiosignal
15
+ Classifier: License :: OSI Approved :: Apache Software License
16
+ Classifier: Intended Audience :: Developers
17
+ Classifier: Programming Language :: Python
18
+ Classifier: Programming Language :: Python :: 3
19
+ Classifier: Programming Language :: Python :: 3 :: Only
20
+ Classifier: Development Status :: 5 - Production/Stable
21
+ Classifier: Operating System :: POSIX
22
+ Classifier: Operating System :: MacOS :: MacOS X
23
+ Classifier: Operating System :: Microsoft :: Windows
24
+ Classifier: Framework :: AsyncIO
25
+ Requires-Python: >=3.9
26
+ Description-Content-Type: text/x-rst
27
+ License-File: LICENSE
28
+ Requires-Dist: frozenlist>=1.1.0
29
+ Requires-Dist: typing-extensions>=4.2; python_version < "3.13"
30
+ Dynamic: license-file
31
+
32
+ =========
33
+ aiosignal
34
+ =========
35
+
36
+ .. image:: https://github.com/aio-libs/aiosignal/workflows/CI/badge.svg
37
+ :target: https://github.com/aio-libs/aiosignal/actions?query=workflow%3ACI
38
+ :alt: GitHub status for master branch
39
+
40
+ .. image:: https://codecov.io/gh/aio-libs/aiosignal/branch/master/graph/badge.svg?flag=pytest
41
+ :target: https://codecov.io/gh/aio-libs/aiosignal?flags[0]=pytest
42
+ :alt: codecov.io status for master branch
43
+
44
+ .. image:: https://badge.fury.io/py/aiosignal.svg
45
+ :target: https://pypi.org/project/aiosignal
46
+ :alt: Latest PyPI package version
47
+
48
+ .. image:: https://readthedocs.org/projects/aiosignal/badge/?version=latest
49
+ :target: https://aiosignal.readthedocs.io/
50
+ :alt: Latest Read The Docs
51
+
52
+ .. image:: https://img.shields.io/discourse/topics?server=https%3A%2F%2Faio-libs.discourse.group%2F
53
+ :target: https://aio-libs.discourse.group/
54
+ :alt: Discourse group for io-libs
55
+
56
+ .. image:: https://badges.gitter.im/Join%20Chat.svg
57
+ :target: https://gitter.im/aio-libs/Lobby
58
+ :alt: Chat on Gitter
59
+
60
+ Introduction
61
+ ============
62
+
63
+ A project to manage callbacks in `asyncio` projects.
64
+
65
+ ``Signal`` is a list of registered asynchronous callbacks.
66
+
67
+ The signal's life-cycle has two stages: after creation its content
68
+ could be filled by using standard list operations: ``sig.append()``
69
+ etc.
70
+
71
+ After you call ``sig.freeze()`` the signal is *frozen*: adding, removing
72
+ and dropping callbacks is forbidden.
73
+
74
+ The only available operation is calling the previously registered
75
+ callbacks by using ``await sig.send(data)``.
76
+
77
+ For concrete usage examples see the `Signals
78
+ <https://docs.aiohttp.org/en/stable/web_advanced.html#aiohttp-web-signals>
79
+ section of the `Web Server Advanced
80
+ <https://docs.aiohttp.org/en/stable/web_advanced.html>` chapter of the `aiohttp
81
+ documentation`_.
82
+
83
+
84
+ Installation
85
+ ------------
86
+
87
+ ::
88
+
89
+ $ pip install aiosignal
90
+
91
+
92
+ Documentation
93
+ =============
94
+
95
+ https://aiosignal.readthedocs.io/
96
+
97
+ License
98
+ =======
99
+
100
+ ``aiosignal`` is offered under the Apache 2 license.
101
+
102
+ Source code
103
+ ===========
104
+
105
+ The project is hosted on GitHub_
106
+
107
+ Please file an issue in the `bug tracker
108
+ <https://github.com/aio-libs/aiosignal/issues>`_ if you have found a bug
109
+ or have some suggestions to improve the library.
110
+
111
+ .. _GitHub: https://github.com/aio-libs/aiosignal
112
+ .. _aiohttp documentation: https://docs.aiohttp.org/
lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/RECORD ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ aiosignal-1.4.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
2
+ aiosignal-1.4.0.dist-info/METADATA,sha256=CSR-8dqLxpZyjUcTDnAuQwf299EB1sSFv_nzpxznAI0,3662
3
+ aiosignal-1.4.0.dist-info/RECORD,,
4
+ aiosignal-1.4.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
5
+ aiosignal-1.4.0.dist-info/licenses/LICENSE,sha256=b9UkPpLdf5jsacesN3co50kFcJ_1J6W_mNbQJjwE9bY,11332
6
+ aiosignal-1.4.0.dist-info/top_level.txt,sha256=z45aNOKGDdrI1roqZY3BGXQ22kJFPHBmVdwtLYLtXC0,10
7
+ aiosignal/__init__.py,sha256=TIkmUG9HTBt4dfq2nISYBiZiRB2xwvFtEZydLP0HPL4,1537
8
+ aiosignal/__pycache__/__init__.cpython-312.pyc,,
9
+ aiosignal/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/WHEEL ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (80.9.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/licenses/LICENSE ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+ 1. Definitions.
8
+
9
+ "License" shall mean the terms and conditions for use, reproduction,
10
+ and distribution as defined by Sections 1 through 9 of this document.
11
+
12
+ "Licensor" shall mean the copyright owner or entity authorized by
13
+ the copyright owner that is granting the License.
14
+
15
+ "Legal Entity" shall mean the union of the acting entity and all
16
+ other entities that control, are controlled by, or are under common
17
+ control with that entity. For the purposes of this definition,
18
+ "control" means (i) the power, direct or indirect, to cause the
19
+ direction or management of such entity, whether by contract or
20
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21
+ outstanding shares, or (iii) beneficial ownership of such entity.
22
+
23
+ "You" (or "Your") shall mean an individual or Legal Entity
24
+ exercising permissions granted by this License.
25
+
26
+ "Source" form shall mean the preferred form for making modifications,
27
+ including but not limited to software source code, documentation
28
+ source, and configuration files.
29
+
30
+ "Object" form shall mean any form resulting from mechanical
31
+ transformation or translation of a Source form, including but
32
+ not limited to compiled object code, generated documentation,
33
+ and conversions to other media types.
34
+
35
+ "Work" shall mean the work of authorship, whether in Source or
36
+ Object form, made available under the License, as indicated by a
37
+ copyright notice that is included in or attached to the work
38
+ (an example is provided in the Appendix below).
39
+
40
+ "Derivative Works" shall mean any work, whether in Source or Object
41
+ form, that is based on (or derived from) the Work and for which the
42
+ editorial revisions, annotations, elaborations, or other modifications
43
+ represent, as a whole, an original work of authorship. For the purposes
44
+ of this License, Derivative Works shall not include works that remain
45
+ separable from, or merely link (or bind by name) to the interfaces of,
46
+ the Work and Derivative Works thereof.
47
+
48
+ "Contribution" shall mean any work of authorship, including
49
+ the original version of the Work and any modifications or additions
50
+ to that Work or Derivative Works thereof, that is intentionally
51
+ submitted to Licensor for inclusion in the Work by the copyright owner
52
+ or by an individual or Legal Entity authorized to submit on behalf of
53
+ the copyright owner. For the purposes of this definition, "submitted"
54
+ means any form of electronic, verbal, or written communication sent
55
+ to the Licensor or its representatives, including but not limited to
56
+ communication on electronic mailing lists, source code control systems,
57
+ and issue tracking systems that are managed by, or on behalf of, the
58
+ Licensor for the purpose of discussing and improving the Work, but
59
+ excluding communication that is conspicuously marked or otherwise
60
+ designated in writing by the copyright owner as "Not a Contribution."
61
+
62
+ "Contributor" shall mean Licensor and any individual or Legal Entity
63
+ on behalf of whom a Contribution has been received by Licensor and
64
+ subsequently incorporated within the Work.
65
+
66
+ 2. Grant of Copyright License. Subject to the terms and conditions of
67
+ this License, each Contributor hereby grants to You a perpetual,
68
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69
+ copyright license to reproduce, prepare Derivative Works of,
70
+ publicly display, publicly perform, sublicense, and distribute the
71
+ Work and such Derivative Works in Source or Object form.
72
+
73
+ 3. Grant of Patent License. Subject to the terms and conditions of
74
+ this License, each Contributor hereby grants to You a perpetual,
75
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76
+ (except as stated in this section) patent license to make, have made,
77
+ use, offer to sell, sell, import, and otherwise transfer the Work,
78
+ where such license applies only to those patent claims licensable
79
+ by such Contributor that are necessarily infringed by their
80
+ Contribution(s) alone or by combination of their Contribution(s)
81
+ with the Work to which such Contribution(s) was submitted. If You
82
+ institute patent litigation against any entity (including a
83
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84
+ or a Contribution incorporated within the Work constitutes direct
85
+ or contributory patent infringement, then any patent licenses
86
+ granted to You under this License for that Work shall terminate
87
+ as of the date such litigation is filed.
88
+
89
+ 4. Redistribution. You may reproduce and distribute copies of the
90
+ Work or Derivative Works thereof in any medium, with or without
91
+ modifications, and in Source or Object form, provided that You
92
+ meet the following conditions:
93
+
94
+ (a) You must give any other recipients of the Work or
95
+ Derivative Works a copy of this License; and
96
+
97
+ (b) You must cause any modified files to carry prominent notices
98
+ stating that You changed the files; and
99
+
100
+ (c) You must retain, in the Source form of any Derivative Works
101
+ that You distribute, all copyright, patent, trademark, and
102
+ attribution notices from the Source form of the Work,
103
+ excluding those notices that do not pertain to any part of
104
+ the Derivative Works; and
105
+
106
+ (d) If the Work includes a "NOTICE" text file as part of its
107
+ distribution, then any Derivative Works that You distribute must
108
+ include a readable copy of the attribution notices contained
109
+ within such NOTICE file, excluding those notices that do not
110
+ pertain to any part of the Derivative Works, in at least one
111
+ of the following places: within a NOTICE text file distributed
112
+ as part of the Derivative Works; within the Source form or
113
+ documentation, if provided along with the Derivative Works; or,
114
+ within a display generated by the Derivative Works, if and
115
+ wherever such third-party notices normally appear. The contents
116
+ of the NOTICE file are for informational purposes only and
117
+ do not modify the License. You may add Your own attribution
118
+ notices within Derivative Works that You distribute, alongside
119
+ or as an addendum to the NOTICE text from the Work, provided
120
+ that such additional attribution notices cannot be construed
121
+ as modifying the License.
122
+
123
+ You may add Your own copyright statement to Your modifications and
124
+ may provide additional or different license terms and conditions
125
+ for use, reproduction, or distribution of Your modifications, or
126
+ for any such Derivative Works as a whole, provided Your use,
127
+ reproduction, and distribution of the Work otherwise complies with
128
+ the conditions stated in this License.
129
+
130
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131
+ any Contribution intentionally submitted for inclusion in the Work
132
+ by You to the Licensor shall be under the terms and conditions of
133
+ this License, without any additional terms or conditions.
134
+ Notwithstanding the above, nothing herein shall supersede or modify
135
+ the terms of any separate license agreement you may have executed
136
+ with Licensor regarding such Contributions.
137
+
138
+ 6. Trademarks. This License does not grant permission to use the trade
139
+ names, trademarks, service marks, or product names of the Licensor,
140
+ except as required for reasonable and customary use in describing the
141
+ origin of the Work and reproducing the content of the NOTICE file.
142
+
143
+ 7. Disclaimer of Warranty. Unless required by applicable law or
144
+ agreed to in writing, Licensor provides the Work (and each
145
+ Contributor provides its Contributions) on an "AS IS" BASIS,
146
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147
+ implied, including, without limitation, any warranties or conditions
148
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149
+ PARTICULAR PURPOSE. You are solely responsible for determining the
150
+ appropriateness of using or redistributing the Work and assume any
151
+ risks associated with Your exercise of permissions under this License.
152
+
153
+ 8. Limitation of Liability. In no event and under no legal theory,
154
+ whether in tort (including negligence), contract, or otherwise,
155
+ unless required by applicable law (such as deliberate and grossly
156
+ negligent acts) or agreed to in writing, shall any Contributor be
157
+ liable to You for damages, including any direct, indirect, special,
158
+ incidental, or consequential damages of any character arising as a
159
+ result of this License or out of the use or inability to use the
160
+ Work (including but not limited to damages for loss of goodwill,
161
+ work stoppage, computer failure or malfunction, or any and all
162
+ other commercial damages or losses), even if such Contributor
163
+ has been advised of the possibility of such damages.
164
+
165
+ 9. Accepting Warranty or Additional Liability. While redistributing
166
+ the Work or Derivative Works thereof, You may choose to offer,
167
+ and charge a fee for, acceptance of support, warranty, indemnity,
168
+ or other liability obligations and/or rights consistent with this
169
+ License. However, in accepting such obligations, You may act only
170
+ on Your own behalf and on Your sole responsibility, not on behalf
171
+ of any other Contributor, and only if You agree to indemnify,
172
+ defend, and hold each Contributor harmless for any liability
173
+ incurred by, or claims asserted against, such Contributor by reason
174
+ of your accepting any such warranty or additional liability.
175
+
176
+ END OF TERMS AND CONDITIONS
177
+
178
+ APPENDIX: How to apply the Apache License to your work.
179
+
180
+ To apply the Apache License to your work, attach the following
181
+ boilerplate notice, with the fields enclosed by brackets "{}"
182
+ replaced with your own identifying information. (Don't include
183
+ the brackets!) The text should be enclosed in the appropriate
184
+ comment syntax for the file format. We also recommend that a
185
+ file or class name and description of purpose be included on the
186
+ same "printed page" as the copyright notice for easier
187
+ identification within third-party archives.
188
+
189
+ Copyright 2013-2019 Nikolay Kim and Andrew Svetlov
190
+
191
+ Licensed under the Apache License, Version 2.0 (the "License");
192
+ you may not use this file except in compliance with the License.
193
+ You may obtain a copy of the License at
194
+
195
+ http://www.apache.org/licenses/LICENSE-2.0
196
+
197
+ Unless required by applicable law or agreed to in writing, software
198
+ distributed under the License is distributed on an "AS IS" BASIS,
199
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
+ See the License for the specific language governing permissions and
201
+ limitations under the License.
lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/top_level.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ aiosignal
lib/python3.12/site-packages/datasets/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (1.45 kB). View file
 
lib/python3.12/site-packages/datasets/__pycache__/arrow_reader.cpython-312.pyc ADDED
Binary file (28.4 kB). View file
 
lib/python3.12/site-packages/datasets/__pycache__/arrow_writer.cpython-312.pyc ADDED
Binary file (34.9 kB). View file
 
lib/python3.12/site-packages/datasets/__pycache__/builder.cpython-312.pyc ADDED
Binary file (97 kB). View file
 
lib/python3.12/site-packages/datasets/__pycache__/combine.cpython-312.pyc ADDED
Binary file (10.8 kB). View file
 
lib/python3.12/site-packages/datasets/__pycache__/config.cpython-312.pyc ADDED
Binary file (11.9 kB). View file
 
lib/python3.12/site-packages/datasets/__pycache__/data_files.cpython-312.pyc ADDED
Binary file (37.1 kB). View file
 
lib/python3.12/site-packages/datasets/__pycache__/distributed.cpython-312.pyc ADDED
Binary file (1.85 kB). View file
 
lib/python3.12/site-packages/datasets/__pycache__/exceptions.cpython-312.pyc ADDED
Binary file (6.65 kB). View file
 
lib/python3.12/site-packages/datasets/__pycache__/fingerprint.cpython-312.pyc ADDED
Binary file (21.6 kB). View file
 
lib/python3.12/site-packages/datasets/__pycache__/hub.cpython-312.pyc ADDED
Binary file (9.9 kB). View file
 
lib/python3.12/site-packages/datasets/__pycache__/info.cpython-312.pyc ADDED
Binary file (24.5 kB). View file
 
lib/python3.12/site-packages/datasets/__pycache__/inspect.cpython-312.pyc ADDED
Binary file (17 kB). View file
 
lib/python3.12/site-packages/datasets/__pycache__/keyhash.cpython-312.pyc ADDED
Binary file (4.71 kB). View file
 
lib/python3.12/site-packages/datasets/__pycache__/load.cpython-312.pyc ADDED
Binary file (99.6 kB). View file
 
lib/python3.12/site-packages/datasets/__pycache__/naming.cpython-312.pyc ADDED
Binary file (3.95 kB). View file
 
lib/python3.12/site-packages/datasets/__pycache__/search.cpython-312.pyc ADDED
Binary file (42.8 kB). View file
 
lib/python3.12/site-packages/datasets/__pycache__/splits.cpython-312.pyc ADDED
Binary file (29.4 kB). View file
 
lib/python3.12/site-packages/datasets/__pycache__/streaming.cpython-312.pyc ADDED
Binary file (7.8 kB). View file
 
lib/python3.12/site-packages/datasets/download/__init__.py ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ __all__ = [
2
+ "DownloadConfig",
3
+ "DownloadManager",
4
+ "DownloadMode",
5
+ "StreamingDownloadManager",
6
+ ]
7
+
8
+ from .download_config import DownloadConfig
9
+ from .download_manager import DownloadManager, DownloadMode
10
+ from .streaming_download_manager import StreamingDownloadManager
lib/python3.12/site-packages/datasets/download/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (457 Bytes). View file
 
lib/python3.12/site-packages/datasets/download/__pycache__/download_config.cpython-312.pyc ADDED
Binary file (5.1 kB). View file
 
lib/python3.12/site-packages/datasets/download/__pycache__/download_manager.cpython-312.pyc ADDED
Binary file (15.1 kB). View file
 
lib/python3.12/site-packages/datasets/download/__pycache__/streaming_download_manager.cpython-312.pyc ADDED
Binary file (9.31 kB). View file
 
lib/python3.12/site-packages/datasets/download/download_config.py ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import copy
2
+ from dataclasses import dataclass, field
3
+ from pathlib import Path
4
+ from typing import Any, Optional, Union
5
+
6
+ from .. import config
7
+
8
+
9
+ @dataclass
10
+ class DownloadConfig:
11
+ """Configuration for our cached path manager.
12
+
13
+ Attributes:
14
+ cache_dir (`str` or `Path`, *optional*):
15
+ Specify a cache directory to save the file to (overwrite the
16
+ default cache dir).
17
+ force_download (`bool`, defaults to `False`):
18
+ If `True`, re-download the file even if it's already cached in
19
+ the cache dir.
20
+ resume_download (`bool`, defaults to `False`):
21
+ If `True`, resume the download if an incompletely received file is
22
+ found.
23
+ proxies (`dict`, *optional*):
24
+ user_agent (`str`, *optional*):
25
+ Optional string or dict that will be appended to the user-agent on remote
26
+ requests.
27
+ extract_compressed_file (`bool`, defaults to `False`):
28
+ If `True` and the path point to a zip or tar file,
29
+ extract the compressed file in a folder along the archive.
30
+ force_extract (`bool`, defaults to `False`):
31
+ If `True` when `extract_compressed_file` is `True` and the archive
32
+ was already extracted, re-extract the archive and override the folder where it was extracted.
33
+ delete_extracted (`bool`, defaults to `False`):
34
+ Whether to delete (or keep) the extracted files.
35
+ extract_on_the_fly (`bool`, defaults to `False`):
36
+ If `True`, extract compressed files while they are being read.
37
+ use_etag (`bool`, defaults to `True`):
38
+ Whether to use the ETag HTTP response header to validate the cached files.
39
+ num_proc (`int`, *optional*):
40
+ The number of processes to launch to download the files in parallel.
41
+ max_retries (`int`, default to `1`):
42
+ The number of times to retry an HTTP request if it fails.
43
+ token (`str` or `bool`, *optional*):
44
+ Optional string or boolean to use as Bearer token
45
+ for remote files on the Datasets Hub. If `True`, or not specified, will get token from `~/.huggingface`.
46
+ storage_options (`dict`, *optional*):
47
+ Key/value pairs to be passed on to the dataset file-system backend, if any.
48
+ download_desc (`str`, *optional*):
49
+ A description to be displayed alongside with the progress bar while downloading the files.
50
+ disable_tqdm (`bool`, defaults to `False`):
51
+ Whether to disable the individual files download progress bar
52
+ """
53
+
54
+ cache_dir: Optional[Union[str, Path]] = None
55
+ force_download: bool = False
56
+ resume_download: bool = False
57
+ local_files_only: bool = False
58
+ proxies: Optional[dict] = None
59
+ user_agent: Optional[str] = None
60
+ extract_compressed_file: bool = False
61
+ force_extract: bool = False
62
+ delete_extracted: bool = False
63
+ extract_on_the_fly: bool = False
64
+ use_etag: bool = True
65
+ num_proc: Optional[int] = None
66
+ max_retries: int = 1
67
+ token: Optional[Union[str, bool]] = None
68
+ storage_options: dict[str, Any] = field(default_factory=dict)
69
+ download_desc: Optional[str] = None
70
+ disable_tqdm: bool = False
71
+
72
+ def copy(self) -> "DownloadConfig":
73
+ return self.__class__(**{k: copy.deepcopy(v) for k, v in self.__dict__.items()})
74
+
75
+ def __setattr__(self, name, value):
76
+ if name == "token" and getattr(self, "storage_options", None) is not None:
77
+ if "hf" not in self.storage_options:
78
+ self.storage_options["hf"] = {"token": value, "endpoint": config.HF_ENDPOINT}
79
+ elif getattr(self.storage_options["hf"], "token", None) is None:
80
+ self.storage_options["hf"]["token"] = value
81
+ super().__setattr__(name, value)
lib/python3.12/site-packages/datasets/download/download_manager.py ADDED
@@ -0,0 +1,340 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2020 The TensorFlow Datasets Authors.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ # Lint as: python3
16
+ """Download manager interface."""
17
+
18
+ import enum
19
+ import io
20
+ import multiprocessing
21
+ import os
22
+ from datetime import datetime
23
+ from functools import partial
24
+ from typing import Optional, Union
25
+
26
+ import fsspec
27
+ from fsspec.core import url_to_fs
28
+ from tqdm.contrib.concurrent import thread_map
29
+
30
+ from .. import config
31
+ from ..utils import tqdm as hf_tqdm
32
+ from ..utils.file_utils import (
33
+ ArchiveIterable,
34
+ FilesIterable,
35
+ cached_path,
36
+ is_relative_path,
37
+ stack_multiprocessing_download_progress_bars,
38
+ url_or_path_join,
39
+ )
40
+ from ..utils.info_utils import get_size_checksum_dict
41
+ from ..utils.logging import get_logger, tqdm
42
+ from ..utils.py_utils import NestedDataStructure, map_nested
43
+ from ..utils.track import tracked_str
44
+ from .download_config import DownloadConfig
45
+
46
+
47
+ logger = get_logger(__name__)
48
+
49
+
50
+ class DownloadMode(enum.Enum):
51
+ """`Enum` for how to treat pre-existing downloads and data.
52
+
53
+ The default mode is `REUSE_DATASET_IF_EXISTS`, which will reuse both
54
+ raw downloads and the prepared dataset if they exist.
55
+
56
+ The generations modes:
57
+
58
+ | | Downloads | Dataset |
59
+ |-------------------------------------|-----------|---------|
60
+ | `REUSE_DATASET_IF_EXISTS` (default) | Reuse | Reuse |
61
+ | `REUSE_CACHE_IF_EXISTS` | Reuse | Fresh |
62
+ | `FORCE_REDOWNLOAD` | Fresh | Fresh |
63
+
64
+ """
65
+
66
+ REUSE_DATASET_IF_EXISTS = "reuse_dataset_if_exists"
67
+ REUSE_CACHE_IF_EXISTS = "reuse_cache_if_exists"
68
+ FORCE_REDOWNLOAD = "force_redownload"
69
+
70
+
71
+ class DownloadManager:
72
+ is_streaming = False
73
+
74
+ def __init__(
75
+ self,
76
+ dataset_name: Optional[str] = None,
77
+ data_dir: Optional[str] = None,
78
+ download_config: Optional[DownloadConfig] = None,
79
+ base_path: Optional[str] = None,
80
+ record_checksums=True,
81
+ ):
82
+ """Download manager constructor.
83
+
84
+ Args:
85
+ data_dir:
86
+ can be used to specify a manual directory to get the files from.
87
+ dataset_name (`str`):
88
+ name of dataset this instance will be used for. If
89
+ provided, downloads will contain which datasets they were used for.
90
+ download_config (`DownloadConfig`):
91
+ to specify the cache directory and other
92
+ download options
93
+ base_path (`str`):
94
+ base path that is used when relative paths are used to
95
+ download files. This can be a remote url.
96
+ record_checksums (`bool`, defaults to `True`):
97
+ Whether to record the checksums of the downloaded files. If None, the value is inferred from the builder.
98
+ """
99
+ self._dataset_name = dataset_name
100
+ self._data_dir = data_dir
101
+ self._base_path = base_path or os.path.abspath(".")
102
+ # To record what is being used: {url: {num_bytes: int, checksum: str}}
103
+ self._recorded_sizes_checksums: dict[str, dict[str, Optional[Union[int, str]]]] = {}
104
+ self.record_checksums = record_checksums
105
+ self.download_config = download_config or DownloadConfig()
106
+ self.downloaded_paths = {}
107
+ self.extracted_paths = {}
108
+
109
+ @property
110
+ def manual_dir(self):
111
+ return self._data_dir
112
+
113
+ @property
114
+ def downloaded_size(self):
115
+ """Returns the total size of downloaded files."""
116
+ return sum(checksums_dict["num_bytes"] for checksums_dict in self._recorded_sizes_checksums.values())
117
+
118
+ def _record_sizes_checksums(self, url_or_urls: NestedDataStructure, downloaded_path_or_paths: NestedDataStructure):
119
+ """Record size/checksum of downloaded files."""
120
+ delay = 5
121
+ for url, path in hf_tqdm(
122
+ list(zip(url_or_urls.flatten(), downloaded_path_or_paths.flatten())),
123
+ delay=delay,
124
+ desc="Computing checksums",
125
+ ):
126
+ # call str to support PathLike objects
127
+ self._recorded_sizes_checksums[str(url)] = get_size_checksum_dict(
128
+ path, record_checksum=self.record_checksums
129
+ )
130
+
131
+ def download(self, url_or_urls):
132
+ """Download given URL(s).
133
+
134
+ By default, only one process is used for download. Pass customized `download_config.num_proc` to change this behavior.
135
+
136
+ Args:
137
+ url_or_urls (`str` or `list` or `dict`):
138
+ URL or `list` or `dict` of URLs to download. Each URL is a `str`.
139
+
140
+ Returns:
141
+ `str` or `list` or `dict`:
142
+ The downloaded paths matching the given input `url_or_urls`.
143
+
144
+ Example:
145
+
146
+ ```py
147
+ >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz')
148
+ ```
149
+ """
150
+ download_config = self.download_config.copy()
151
+ download_config.extract_compressed_file = False
152
+ if download_config.download_desc is None:
153
+ download_config.download_desc = "Downloading data"
154
+
155
+ download_func = partial(self._download_batched, download_config=download_config)
156
+
157
+ start_time = datetime.now()
158
+ with stack_multiprocessing_download_progress_bars():
159
+ downloaded_path_or_paths = map_nested(
160
+ download_func,
161
+ url_or_urls,
162
+ map_tuple=True,
163
+ num_proc=download_config.num_proc,
164
+ desc="Downloading data files",
165
+ batched=True,
166
+ batch_size=-1,
167
+ )
168
+ duration = datetime.now() - start_time
169
+ logger.info(f"Downloading took {duration.total_seconds() // 60} min")
170
+ url_or_urls = NestedDataStructure(url_or_urls)
171
+ downloaded_path_or_paths = NestedDataStructure(downloaded_path_or_paths)
172
+ self.downloaded_paths.update(dict(zip(url_or_urls.flatten(), downloaded_path_or_paths.flatten())))
173
+
174
+ start_time = datetime.now()
175
+ self._record_sizes_checksums(url_or_urls, downloaded_path_or_paths)
176
+ duration = datetime.now() - start_time
177
+ logger.info(f"Checksum Computation took {duration.total_seconds() // 60} min")
178
+
179
+ return downloaded_path_or_paths.data
180
+
181
+ def _download_batched(
182
+ self,
183
+ url_or_filenames: list[str],
184
+ download_config: DownloadConfig,
185
+ ) -> list[str]:
186
+ if len(url_or_filenames) >= 16:
187
+ download_config = download_config.copy()
188
+ download_config.disable_tqdm = True
189
+ download_func = partial(self._download_single, download_config=download_config)
190
+
191
+ fs: fsspec.AbstractFileSystem
192
+ path = str(url_or_filenames[0])
193
+ if is_relative_path(path):
194
+ # append the relative path to the base_path
195
+ path = url_or_path_join(self._base_path, path)
196
+ fs, path = url_to_fs(path, **download_config.storage_options)
197
+ size = 0
198
+ try:
199
+ size = fs.info(path).get("size", 0)
200
+ except Exception:
201
+ pass
202
+ max_workers = (
203
+ config.HF_DATASETS_MULTITHREADING_MAX_WORKERS if size < (20 << 20) else 1
204
+ ) # enable multithreading if files are small
205
+
206
+ return thread_map(
207
+ download_func,
208
+ url_or_filenames,
209
+ desc=download_config.download_desc or "Downloading",
210
+ unit="files",
211
+ position=multiprocessing.current_process()._identity[-1] # contains the ranks of subprocesses
212
+ if os.environ.get("HF_DATASETS_STACK_MULTIPROCESSING_DOWNLOAD_PROGRESS_BARS") == "1"
213
+ and multiprocessing.current_process()._identity
214
+ else None,
215
+ max_workers=max_workers,
216
+ tqdm_class=tqdm,
217
+ )
218
+ else:
219
+ return [
220
+ self._download_single(url_or_filename, download_config=download_config)
221
+ for url_or_filename in url_or_filenames
222
+ ]
223
+
224
+ def _download_single(self, url_or_filename: str, download_config: DownloadConfig) -> str:
225
+ url_or_filename = str(url_or_filename)
226
+ if is_relative_path(url_or_filename):
227
+ # append the relative path to the base_path
228
+ url_or_filename = url_or_path_join(self._base_path, url_or_filename)
229
+ out = cached_path(url_or_filename, download_config=download_config)
230
+ out = tracked_str(out)
231
+ out.set_origin(url_or_filename)
232
+ return out
233
+
234
+ def iter_archive(self, path_or_buf: Union[str, io.BufferedReader]):
235
+ """Iterate over files within an archive.
236
+
237
+ Args:
238
+ path_or_buf (`str` or `io.BufferedReader`):
239
+ Archive path or archive binary file object.
240
+
241
+ Yields:
242
+ `tuple[str, io.BufferedReader]`:
243
+ 2-tuple (path_within_archive, file_object).
244
+ File object is opened in binary mode.
245
+
246
+ Example:
247
+
248
+ ```py
249
+ >>> archive = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz')
250
+ >>> files = dl_manager.iter_archive(archive)
251
+ ```
252
+ """
253
+
254
+ if hasattr(path_or_buf, "read"):
255
+ return ArchiveIterable.from_buf(path_or_buf)
256
+ else:
257
+ return ArchiveIterable.from_urlpath(path_or_buf)
258
+
259
+ def iter_files(self, paths: Union[str, list[str]]):
260
+ """Iterate over file paths.
261
+
262
+ Args:
263
+ paths (`str` or `list` of `str`):
264
+ Root paths.
265
+
266
+ Yields:
267
+ `str`: File path.
268
+
269
+ Example:
270
+
271
+ ```py
272
+ >>> files = dl_manager.download_and_extract('https://huggingface.co/datasets/beans/resolve/main/data/train.zip')
273
+ >>> files = dl_manager.iter_files(files)
274
+ ```
275
+ """
276
+ return FilesIterable.from_urlpaths(paths)
277
+
278
+ def extract(self, path_or_paths):
279
+ """Extract given path(s).
280
+
281
+ Args:
282
+ path_or_paths (path or `list` or `dict`):
283
+ Path of file to extract. Each path is a `str`.
284
+
285
+ Returns:
286
+ extracted_path(s): `str`, The extracted paths matching the given input
287
+ path_or_paths.
288
+
289
+ Example:
290
+
291
+ ```py
292
+ >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz')
293
+ >>> extracted_files = dl_manager.extract(downloaded_files)
294
+ ```
295
+ """
296
+ download_config = self.download_config.copy()
297
+ download_config.extract_compressed_file = True
298
+ extract_func = partial(self._download_single, download_config=download_config)
299
+ extracted_paths = map_nested(
300
+ extract_func,
301
+ path_or_paths,
302
+ num_proc=download_config.num_proc,
303
+ desc="Extracting data files",
304
+ )
305
+ path_or_paths = NestedDataStructure(path_or_paths)
306
+ extracted_paths = NestedDataStructure(extracted_paths)
307
+ self.extracted_paths.update(dict(zip(path_or_paths.flatten(), extracted_paths.flatten())))
308
+ return extracted_paths.data
309
+
310
+ def download_and_extract(self, url_or_urls):
311
+ """Download and extract given `url_or_urls`.
312
+
313
+ Is roughly equivalent to:
314
+
315
+ ```
316
+ extracted_paths = dl_manager.extract(dl_manager.download(url_or_urls))
317
+ ```
318
+
319
+ Args:
320
+ url_or_urls (`str` or `list` or `dict`):
321
+ URL or `list` or `dict` of URLs to download and extract. Each URL is a `str`.
322
+
323
+ Returns:
324
+ extracted_path(s): `str`, extracted paths of given URL(s).
325
+ """
326
+ return self.extract(self.download(url_or_urls))
327
+
328
+ def get_recorded_sizes_checksums(self):
329
+ return self._recorded_sizes_checksums.copy()
330
+
331
+ def delete_extracted_files(self):
332
+ paths_to_delete = set(self.extracted_paths.values()) - set(self.downloaded_paths.values())
333
+ for key, path in list(self.extracted_paths.items()):
334
+ if path in paths_to_delete and os.path.isfile(path):
335
+ os.remove(path)
336
+ del self.extracted_paths[key]
337
+
338
+ def manage_extracted_files(self):
339
+ if self.download_config.delete_extracted:
340
+ self.delete_extracted_files()
lib/python3.12/site-packages/datasets/download/streaming_download_manager.py ADDED
@@ -0,0 +1,219 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import io
2
+ import os
3
+ from collections.abc import Iterable
4
+ from typing import Optional, Union
5
+
6
+ from ..utils.file_utils import ( # noqa: F401 # backward compatibility
7
+ SINGLE_FILE_COMPRESSION_PROTOCOLS,
8
+ ArchiveIterable,
9
+ FilesIterable,
10
+ _get_extraction_protocol,
11
+ _get_path_extension,
12
+ _prepare_path_and_storage_options,
13
+ is_relative_path,
14
+ url_or_path_join,
15
+ xbasename,
16
+ xdirname,
17
+ xet_parse,
18
+ xexists,
19
+ xgetsize,
20
+ xglob,
21
+ xgzip_open,
22
+ xisdir,
23
+ xisfile,
24
+ xjoin,
25
+ xlistdir,
26
+ xnumpy_load,
27
+ xopen,
28
+ xpandas_read_csv,
29
+ xpandas_read_excel,
30
+ xPath,
31
+ xpyarrow_parquet_read_table,
32
+ xrelpath,
33
+ xsio_loadmat,
34
+ xsplit,
35
+ xsplitext,
36
+ xwalk,
37
+ xxml_dom_minidom_parse,
38
+ )
39
+ from ..utils.logging import get_logger
40
+ from ..utils.py_utils import map_nested
41
+ from .download_config import DownloadConfig
42
+
43
+
44
+ logger = get_logger(__name__)
45
+
46
+
47
+ class StreamingDownloadManager:
48
+ """
49
+ Download manager that uses the "::" separator to navigate through (possibly remote) compressed archives.
50
+ Contrary to the regular `DownloadManager`, the `download` and `extract` methods don't actually download nor extract
51
+ data, but they rather return the path or url that could be opened using the `xopen` function which extends the
52
+ built-in `open` function to stream data from remote files.
53
+ """
54
+
55
+ is_streaming = True
56
+
57
+ def __init__(
58
+ self,
59
+ dataset_name: Optional[str] = None,
60
+ data_dir: Optional[str] = None,
61
+ download_config: Optional[DownloadConfig] = None,
62
+ base_path: Optional[str] = None,
63
+ ):
64
+ self._dataset_name = dataset_name
65
+ self._data_dir = data_dir
66
+ self._base_path = base_path or os.path.abspath(".")
67
+ self.download_config = download_config or DownloadConfig()
68
+ self.downloaded_size = None
69
+ self.record_checksums = False
70
+
71
+ @property
72
+ def manual_dir(self):
73
+ return self._data_dir
74
+
75
+ def download(self, url_or_urls):
76
+ """Normalize URL(s) of files to stream data from.
77
+ This is the lazy version of `DownloadManager.download` for streaming.
78
+
79
+ Args:
80
+ url_or_urls (`str` or `list` or `dict`):
81
+ URL(s) of files to stream data from. Each url is a `str`.
82
+
83
+ Returns:
84
+ url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input url_or_urls.
85
+
86
+ Example:
87
+
88
+ ```py
89
+ >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz')
90
+ ```
91
+ """
92
+ url_or_urls = map_nested(self._download_single, url_or_urls, map_tuple=True)
93
+ return url_or_urls
94
+
95
+ def _download_single(self, urlpath: str) -> str:
96
+ urlpath = str(urlpath)
97
+ if is_relative_path(urlpath):
98
+ # append the relative path to the base_path
99
+ urlpath = url_or_path_join(self._base_path, urlpath)
100
+ return urlpath
101
+
102
+ def extract(self, url_or_urls):
103
+ """Add extraction protocol for given url(s) for streaming.
104
+
105
+ This is the lazy version of `DownloadManager.extract` for streaming.
106
+
107
+ Args:
108
+ url_or_urls (`str` or `list` or `dict`):
109
+ URL(s) of files to stream data from. Each url is a `str`.
110
+
111
+ Returns:
112
+ url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input `url_or_urls`.
113
+
114
+ Example:
115
+
116
+ ```py
117
+ >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz')
118
+ >>> extracted_files = dl_manager.extract(downloaded_files)
119
+ ```
120
+ """
121
+ urlpaths = map_nested(self._extract, url_or_urls, map_tuple=True)
122
+ return urlpaths
123
+
124
+ def _extract(self, urlpath: str) -> str:
125
+ urlpath = str(urlpath)
126
+ protocol = _get_extraction_protocol(urlpath, download_config=self.download_config)
127
+ # get inner file: zip://train-00000.json.gz::https://foo.bar/data.zip -> zip://train-00000.json.gz
128
+ path = urlpath.split("::")[0]
129
+ extension = _get_path_extension(path)
130
+ if extension in ["tgz", "tar"] or path.endswith((".tar.gz", ".tar.bz2", ".tar.xz")):
131
+ raise NotImplementedError(
132
+ f"Extraction protocol for TAR archives like '{urlpath}' is not implemented in streaming mode. "
133
+ f"Please use `dl_manager.iter_archive` instead.\n\n"
134
+ f"Example usage:\n\n"
135
+ f"\turl = dl_manager.download(url)\n"
136
+ f"\ttar_archive_iterator = dl_manager.iter_archive(url)\n\n"
137
+ f"\tfor filename, file in tar_archive_iterator:\n"
138
+ f"\t\t..."
139
+ )
140
+ if protocol is None:
141
+ # no extraction
142
+ return urlpath
143
+ elif protocol in SINGLE_FILE_COMPRESSION_PROTOCOLS:
144
+ # there is one single file which is the uncompressed file
145
+ inner_file = os.path.basename(urlpath.split("::")[0])
146
+ inner_file = inner_file[: inner_file.rindex(".")] if "." in inner_file else inner_file
147
+ return f"{protocol}://{inner_file}::{urlpath}"
148
+ else:
149
+ return f"{protocol}://::{urlpath}"
150
+
151
+ def download_and_extract(self, url_or_urls):
152
+ """Prepare given `url_or_urls` for streaming (add extraction protocol).
153
+
154
+ This is the lazy version of `DownloadManager.download_and_extract` for streaming.
155
+
156
+ Is equivalent to:
157
+
158
+ ```
159
+ urls = dl_manager.extract(dl_manager.download(url_or_urls))
160
+ ```
161
+
162
+ Args:
163
+ url_or_urls (`str` or `list` or `dict`):
164
+ URL(s) to stream from data from. Each url is a `str`.
165
+
166
+ Returns:
167
+ url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input `url_or_urls`.
168
+ """
169
+ return self.extract(self.download(url_or_urls))
170
+
171
+ def iter_archive(self, urlpath_or_buf: Union[str, io.BufferedReader]) -> Iterable[tuple]:
172
+ """Iterate over files within an archive.
173
+
174
+ Args:
175
+ urlpath_or_buf (`str` or `io.BufferedReader`):
176
+ Archive path or archive binary file object.
177
+
178
+ Yields:
179
+ `tuple[str, io.BufferedReader]`:
180
+ 2-tuple (path_within_archive, file_object).
181
+ File object is opened in binary mode.
182
+
183
+ Example:
184
+
185
+ ```py
186
+ >>> archive = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz')
187
+ >>> files = dl_manager.iter_archive(archive)
188
+ ```
189
+ """
190
+
191
+ if hasattr(urlpath_or_buf, "read"):
192
+ return ArchiveIterable.from_buf(urlpath_or_buf)
193
+ else:
194
+ return ArchiveIterable.from_urlpath(urlpath_or_buf, download_config=self.download_config)
195
+
196
+ def iter_files(self, urlpaths: Union[str, list[str]]) -> Iterable[str]:
197
+ """Iterate over files.
198
+
199
+ Args:
200
+ urlpaths (`str` or `list` of `str`):
201
+ Root paths.
202
+
203
+ Yields:
204
+ str: File URL path.
205
+
206
+ Example:
207
+
208
+ ```py
209
+ >>> files = dl_manager.download_and_extract('https://huggingface.co/datasets/beans/resolve/main/data/train.zip')
210
+ >>> files = dl_manager.iter_files(files)
211
+ ```
212
+ """
213
+ return FilesIterable.from_urlpaths(urlpaths, download_config=self.download_config)
214
+
215
+ def manage_extracted_files(self):
216
+ pass
217
+
218
+ def get_recorded_sizes_checksums(self):
219
+ pass
lib/python3.12/site-packages/datasets/features/__init__.py ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ __all__ = [
2
+ "Audio",
3
+ "Array2D",
4
+ "Array3D",
5
+ "Array4D",
6
+ "Array5D",
7
+ "ClassLabel",
8
+ "Features",
9
+ "LargeList",
10
+ "Sequence",
11
+ "Value",
12
+ "Image",
13
+ "Translation",
14
+ "TranslationVariableLanguages",
15
+ "Video",
16
+ "Pdf",
17
+ ]
18
+ from .audio import Audio
19
+ from .features import Array2D, Array3D, Array4D, Array5D, ClassLabel, Features, LargeList, Sequence, Value
20
+ from .image import Image
21
+ from .pdf import Pdf
22
+ from .translation import Translation, TranslationVariableLanguages
23
+ from .video import Video
lib/python3.12/site-packages/datasets/features/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (724 Bytes). View file
 
lib/python3.12/site-packages/datasets/features/__pycache__/audio.cpython-312.pyc ADDED
Binary file (15.4 kB). View file
 
lib/python3.12/site-packages/datasets/features/__pycache__/image.cpython-312.pyc ADDED
Binary file (19.9 kB). View file
 
lib/python3.12/site-packages/datasets/features/__pycache__/pdf.cpython-312.pyc ADDED
Binary file (11.5 kB). View file
 
lib/python3.12/site-packages/datasets/features/__pycache__/translation.cpython-312.pyc ADDED
Binary file (6.47 kB). View file
 
lib/python3.12/site-packages/datasets/features/__pycache__/video.cpython-312.pyc ADDED
Binary file (13.8 kB). View file
 
lib/python3.12/site-packages/datasets/features/audio.py ADDED
@@ -0,0 +1,274 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from dataclasses import dataclass, field
3
+ from io import BytesIO
4
+ from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union
5
+
6
+ import numpy as np
7
+ import pyarrow as pa
8
+
9
+ from .. import config
10
+ from ..download.download_config import DownloadConfig
11
+ from ..table import array_cast
12
+ from ..utils.file_utils import xopen, xsplitext
13
+ from ..utils.py_utils import no_op_if_value_is_null, string_to_dict
14
+
15
+
16
+ if TYPE_CHECKING:
17
+ from .features import FeatureType
18
+
19
+
20
+ @dataclass
21
+ class Audio:
22
+ """Audio [`Feature`] to extract audio data from an audio file.
23
+
24
+ Input: The Audio feature accepts as input:
25
+ - A `str`: Absolute path to the audio file (i.e. random access is allowed).
26
+ - A `dict` with the keys:
27
+
28
+ - `path`: String with relative path of the audio file to the archive file.
29
+ - `bytes`: Bytes content of the audio file.
30
+
31
+ This is useful for archived files with sequential access.
32
+
33
+ - A `dict` with the keys:
34
+
35
+ - `path`: String with relative path of the audio file to the archive file.
36
+ - `array`: Array containing the audio sample
37
+ - `sampling_rate`: Integer corresponding to the sampling rate of the audio sample.
38
+
39
+ This is useful for archived files with sequential access.
40
+
41
+ Args:
42
+ sampling_rate (`int`, *optional*):
43
+ Target sampling rate. If `None`, the native sampling rate is used.
44
+ mono (`bool`, defaults to `True`):
45
+ Whether to convert the audio signal to mono by averaging samples across
46
+ channels.
47
+ decode (`bool`, defaults to `True`):
48
+ Whether to decode the audio data. If `False`,
49
+ returns the underlying dictionary in the format `{"path": audio_path, "bytes": audio_bytes}`.
50
+
51
+ Example:
52
+
53
+ ```py
54
+ >>> from datasets import load_dataset, Audio
55
+ >>> ds = load_dataset("PolyAI/minds14", name="en-US", split="train")
56
+ >>> ds = ds.cast_column("audio", Audio(sampling_rate=16000))
57
+ >>> ds[0]["audio"]
58
+ {'array': array([ 2.3443763e-05, 2.1729663e-04, 2.2145823e-04, ...,
59
+ 3.8356509e-05, -7.3497440e-06, -2.1754686e-05], dtype=float32),
60
+ 'path': '/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav',
61
+ 'sampling_rate': 16000}
62
+ ```
63
+ """
64
+
65
+ sampling_rate: Optional[int] = None
66
+ mono: bool = True
67
+ decode: bool = True
68
+ id: Optional[str] = None
69
+ # Automatically constructed
70
+ dtype: ClassVar[str] = "dict"
71
+ pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()})
72
+ _type: str = field(default="Audio", init=False, repr=False)
73
+
74
+ def __call__(self):
75
+ return self.pa_type
76
+
77
+ def encode_example(self, value: Union[str, bytes, bytearray, dict]) -> dict:
78
+ """Encode example into a format for Arrow.
79
+
80
+ Args:
81
+ value (`str` or `dict`):
82
+ Data passed as input to Audio feature.
83
+
84
+ Returns:
85
+ `dict`
86
+ """
87
+ try:
88
+ import soundfile as sf # soundfile is a dependency of librosa, needed to decode audio files.
89
+ except ImportError as err:
90
+ raise ImportError("To support encoding audio data, please install 'soundfile'.") from err
91
+ if isinstance(value, str):
92
+ return {"bytes": None, "path": value}
93
+ elif isinstance(value, (bytes, bytearray)):
94
+ return {"bytes": value, "path": None}
95
+ elif "array" in value:
96
+ # convert the audio array to wav bytes
97
+ buffer = BytesIO()
98
+ sf.write(buffer, value["array"], value["sampling_rate"], format="wav")
99
+ return {"bytes": buffer.getvalue(), "path": None}
100
+ elif value.get("path") is not None and os.path.isfile(value["path"]):
101
+ # we set "bytes": None to not duplicate the data if they're already available locally
102
+ if value["path"].endswith("pcm"):
103
+ # "PCM" only has raw audio bytes
104
+ if value.get("sampling_rate") is None:
105
+ # At least, If you want to convert "PCM-byte" to "WAV-byte", you have to know sampling rate
106
+ raise KeyError("To use PCM files, please specify a 'sampling_rate' in Audio object")
107
+ if value.get("bytes"):
108
+ # If we already had PCM-byte, we don`t have to make "read file, make bytes" (just use it!)
109
+ bytes_value = np.frombuffer(value["bytes"], dtype=np.int16).astype(np.float32) / 32767
110
+ else:
111
+ bytes_value = np.memmap(value["path"], dtype="h", mode="r").astype(np.float32) / 32767
112
+
113
+ buffer = BytesIO(b"")
114
+ sf.write(buffer, bytes_value, value["sampling_rate"], format="wav")
115
+ return {"bytes": buffer.getvalue(), "path": None}
116
+ else:
117
+ return {"bytes": None, "path": value.get("path")}
118
+ elif value.get("bytes") is not None or value.get("path") is not None:
119
+ # store the audio bytes, and path is used to infer the audio format using the file extension
120
+ return {"bytes": value.get("bytes"), "path": value.get("path")}
121
+ else:
122
+ raise ValueError(
123
+ f"An audio sample should have one of 'path' or 'bytes' but they are missing or None in {value}."
124
+ )
125
+
126
+ def decode_example(
127
+ self, value: dict, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None
128
+ ) -> dict:
129
+ """Decode example audio file into audio data.
130
+
131
+ Args:
132
+ value (`dict`):
133
+ A dictionary with keys:
134
+
135
+ - `path`: String with relative audio file path.
136
+ - `bytes`: Bytes of the audio file.
137
+ token_per_repo_id (`dict`, *optional*):
138
+ To access and decode
139
+ audio files from private repositories on the Hub, you can pass
140
+ a dictionary repo_id (`str`) -> token (`bool` or `str`)
141
+
142
+ Returns:
143
+ `dict`
144
+ """
145
+ if not self.decode:
146
+ raise RuntimeError("Decoding is disabled for this feature. Please use Audio(decode=True) instead.")
147
+
148
+ path, file = (value["path"], BytesIO(value["bytes"])) if value["bytes"] is not None else (value["path"], None)
149
+ if path is None and file is None:
150
+ raise ValueError(f"An audio sample should have one of 'path' or 'bytes' but both are None in {value}.")
151
+
152
+ try:
153
+ import librosa
154
+ import soundfile as sf
155
+ except ImportError as err:
156
+ raise ImportError("To support decoding audio files, please install 'librosa' and 'soundfile'.") from err
157
+
158
+ audio_format = xsplitext(path)[1][1:].lower() if path is not None else None
159
+ if not config.IS_OPUS_SUPPORTED and audio_format == "opus":
160
+ raise RuntimeError(
161
+ "Decoding 'opus' files requires system library 'libsndfile'>=1.0.31, "
162
+ 'You can try to update `soundfile` python library: `pip install "soundfile>=0.12.1"`. '
163
+ )
164
+ elif not config.IS_MP3_SUPPORTED and audio_format == "mp3":
165
+ raise RuntimeError(
166
+ "Decoding 'mp3' files requires system library 'libsndfile'>=1.1.0, "
167
+ 'You can try to update `soundfile` python library: `pip install "soundfile>=0.12.1"`. '
168
+ )
169
+
170
+ if file is None:
171
+ token_per_repo_id = token_per_repo_id or {}
172
+ source_url = path.split("::")[-1]
173
+ pattern = (
174
+ config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL
175
+ )
176
+ source_url_fields = string_to_dict(source_url, pattern)
177
+ token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None
178
+
179
+ download_config = DownloadConfig(token=token)
180
+ with xopen(path, "rb", download_config=download_config) as f:
181
+ array, sampling_rate = sf.read(f)
182
+
183
+ else:
184
+ array, sampling_rate = sf.read(file)
185
+
186
+ array = array.T
187
+ if self.mono:
188
+ array = librosa.to_mono(array)
189
+ if self.sampling_rate and self.sampling_rate != sampling_rate:
190
+ array = librosa.resample(array, orig_sr=sampling_rate, target_sr=self.sampling_rate)
191
+ sampling_rate = self.sampling_rate
192
+
193
+ return {"path": path, "array": array, "sampling_rate": sampling_rate}
194
+
195
+ def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]:
196
+ """If in the decodable state, raise an error, otherwise flatten the feature into a dictionary."""
197
+ from .features import Value
198
+
199
+ if self.decode:
200
+ raise ValueError("Cannot flatten a decoded Audio feature.")
201
+ return {
202
+ "bytes": Value("binary"),
203
+ "path": Value("string"),
204
+ }
205
+
206
+ def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray]) -> pa.StructArray:
207
+ """Cast an Arrow array to the Audio arrow storage type.
208
+ The Arrow types that can be converted to the Audio pyarrow storage type are:
209
+
210
+ - `pa.string()` - it must contain the "path" data
211
+ - `pa.binary()` - it must contain the audio bytes
212
+ - `pa.struct({"bytes": pa.binary()})`
213
+ - `pa.struct({"path": pa.string()})`
214
+ - `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter
215
+
216
+ Args:
217
+ storage (`Union[pa.StringArray, pa.StructArray]`):
218
+ PyArrow array to cast.
219
+
220
+ Returns:
221
+ `pa.StructArray`: Array in the Audio arrow storage type, that is
222
+ `pa.struct({"bytes": pa.binary(), "path": pa.string()})`
223
+ """
224
+ if pa.types.is_string(storage.type):
225
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
226
+ storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null())
227
+ elif pa.types.is_binary(storage.type):
228
+ path_array = pa.array([None] * len(storage), type=pa.string())
229
+ storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null())
230
+ elif pa.types.is_struct(storage.type) and storage.type.get_all_field_indices("array"):
231
+ storage = pa.array([Audio().encode_example(x) if x is not None else None for x in storage.to_pylist()])
232
+ elif pa.types.is_struct(storage.type):
233
+ if storage.type.get_field_index("bytes") >= 0:
234
+ bytes_array = storage.field("bytes")
235
+ else:
236
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
237
+ if storage.type.get_field_index("path") >= 0:
238
+ path_array = storage.field("path")
239
+ else:
240
+ path_array = pa.array([None] * len(storage), type=pa.string())
241
+ storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null())
242
+ return array_cast(storage, self.pa_type)
243
+
244
+ def embed_storage(self, storage: pa.StructArray) -> pa.StructArray:
245
+ """Embed audio files into the Arrow array.
246
+
247
+ Args:
248
+ storage (`pa.StructArray`):
249
+ PyArrow array to embed.
250
+
251
+ Returns:
252
+ `pa.StructArray`: Array in the Audio arrow storage type, that is
253
+ `pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
254
+ """
255
+
256
+ @no_op_if_value_is_null
257
+ def path_to_bytes(path):
258
+ with xopen(path, "rb") as f:
259
+ bytes_ = f.read()
260
+ return bytes_
261
+
262
+ bytes_array = pa.array(
263
+ [
264
+ (path_to_bytes(x["path"]) if x["bytes"] is None else x["bytes"]) if x is not None else None
265
+ for x in storage.to_pylist()
266
+ ],
267
+ type=pa.binary(),
268
+ )
269
+ path_array = pa.array(
270
+ [os.path.basename(path) if path is not None else None for path in storage.field("path").to_pylist()],
271
+ type=pa.string(),
272
+ )
273
+ storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=bytes_array.is_null())
274
+ return array_cast(storage, self.pa_type)
lib/python3.12/site-packages/datasets/features/features.py ADDED
@@ -0,0 +1,2332 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ # Lint as: python3
16
+ """This class handle features definition in datasets and some utilities to display table type."""
17
+
18
+ import copy
19
+ import json
20
+ import re
21
+ import sys
22
+ from collections.abc import Iterable, Mapping
23
+ from collections.abc import Sequence as SequenceABC
24
+ from collections.abc import Sequence as Sequence_
25
+ from dataclasses import InitVar, dataclass, field, fields
26
+ from functools import reduce, wraps
27
+ from operator import mul
28
+ from typing import Any, Callable, ClassVar, Literal, Optional, Union
29
+
30
+ import numpy as np
31
+ import pandas as pd
32
+ import pyarrow as pa
33
+ import pyarrow.compute as pc
34
+ import pyarrow.types
35
+ from pandas.api.extensions import ExtensionArray as PandasExtensionArray
36
+ from pandas.api.extensions import ExtensionDtype as PandasExtensionDtype
37
+
38
+ from .. import config
39
+ from ..naming import camelcase_to_snakecase, snakecase_to_camelcase
40
+ from ..table import array_cast
41
+ from ..utils import experimental, logging
42
+ from ..utils.py_utils import asdict, first_non_null_value, zip_dict
43
+ from .audio import Audio
44
+ from .image import Image, encode_pil_image
45
+ from .pdf import Pdf, encode_pdfplumber_pdf
46
+ from .translation import Translation, TranslationVariableLanguages
47
+ from .video import Video
48
+
49
+
50
+ logger = logging.get_logger(__name__)
51
+
52
+
53
+ def _arrow_to_datasets_dtype(arrow_type: pa.DataType) -> str:
54
+ """
55
+ _arrow_to_datasets_dtype takes a pyarrow.DataType and converts it to a datasets string dtype.
56
+ In effect, `dt == string_to_arrow(_arrow_to_datasets_dtype(dt))`
57
+ """
58
+ if pyarrow.types.is_null(arrow_type):
59
+ return "null"
60
+ elif pyarrow.types.is_boolean(arrow_type):
61
+ return "bool"
62
+ elif pyarrow.types.is_int8(arrow_type):
63
+ return "int8"
64
+ elif pyarrow.types.is_int16(arrow_type):
65
+ return "int16"
66
+ elif pyarrow.types.is_int32(arrow_type):
67
+ return "int32"
68
+ elif pyarrow.types.is_int64(arrow_type):
69
+ return "int64"
70
+ elif pyarrow.types.is_uint8(arrow_type):
71
+ return "uint8"
72
+ elif pyarrow.types.is_uint16(arrow_type):
73
+ return "uint16"
74
+ elif pyarrow.types.is_uint32(arrow_type):
75
+ return "uint32"
76
+ elif pyarrow.types.is_uint64(arrow_type):
77
+ return "uint64"
78
+ elif pyarrow.types.is_float16(arrow_type):
79
+ return "float16" # pyarrow dtype is "halffloat"
80
+ elif pyarrow.types.is_float32(arrow_type):
81
+ return "float32" # pyarrow dtype is "float"
82
+ elif pyarrow.types.is_float64(arrow_type):
83
+ return "float64" # pyarrow dtype is "double"
84
+ elif pyarrow.types.is_time32(arrow_type):
85
+ return f"time32[{pa.type_for_alias(str(arrow_type)).unit}]"
86
+ elif pyarrow.types.is_time64(arrow_type):
87
+ return f"time64[{pa.type_for_alias(str(arrow_type)).unit}]"
88
+ elif pyarrow.types.is_timestamp(arrow_type):
89
+ if arrow_type.tz is None:
90
+ return f"timestamp[{arrow_type.unit}]"
91
+ elif arrow_type.tz:
92
+ return f"timestamp[{arrow_type.unit}, tz={arrow_type.tz}]"
93
+ else:
94
+ raise ValueError(f"Unexpected timestamp object {arrow_type}.")
95
+ elif pyarrow.types.is_date32(arrow_type):
96
+ return "date32" # pyarrow dtype is "date32[day]"
97
+ elif pyarrow.types.is_date64(arrow_type):
98
+ return "date64" # pyarrow dtype is "date64[ms]"
99
+ elif pyarrow.types.is_duration(arrow_type):
100
+ return f"duration[{arrow_type.unit}]"
101
+ elif pyarrow.types.is_decimal128(arrow_type):
102
+ return f"decimal128({arrow_type.precision}, {arrow_type.scale})"
103
+ elif pyarrow.types.is_decimal256(arrow_type):
104
+ return f"decimal256({arrow_type.precision}, {arrow_type.scale})"
105
+ elif pyarrow.types.is_binary(arrow_type):
106
+ return "binary"
107
+ elif pyarrow.types.is_large_binary(arrow_type):
108
+ return "large_binary"
109
+ elif pyarrow.types.is_string(arrow_type):
110
+ return "string"
111
+ elif pyarrow.types.is_large_string(arrow_type):
112
+ return "large_string"
113
+ elif pyarrow.types.is_dictionary(arrow_type):
114
+ return _arrow_to_datasets_dtype(arrow_type.value_type)
115
+ else:
116
+ raise ValueError(f"Arrow type {arrow_type} does not have a datasets dtype equivalent.")
117
+
118
+
119
+ def string_to_arrow(datasets_dtype: str) -> pa.DataType:
120
+ """
121
+ string_to_arrow takes a datasets string dtype and converts it to a pyarrow.DataType.
122
+
123
+ In effect, `dt == string_to_arrow(_arrow_to_datasets_dtype(dt))`
124
+
125
+ This is necessary because the datasets.Value() primitive type is constructed using a string dtype
126
+
127
+ Value(dtype=str)
128
+
129
+ But Features.type (via `get_nested_type()` expects to resolve Features into a pyarrow Schema,
130
+ which means that each Value() must be able to resolve into a corresponding pyarrow.DataType, which is the
131
+ purpose of this function.
132
+ """
133
+
134
+ def _dtype_error_msg(dtype, pa_dtype, examples=None, urls=None):
135
+ msg = f"{dtype} is not a validly formatted string representation of the pyarrow {pa_dtype} type."
136
+ if examples:
137
+ examples = ", ".join(examples[:-1]) + " or " + examples[-1] if len(examples) > 1 else examples[0]
138
+ msg += f"\nValid examples include: {examples}."
139
+ if urls:
140
+ urls = ", ".join(urls[:-1]) + " and " + urls[-1] if len(urls) > 1 else urls[0]
141
+ msg += f"\nFor more insformation, see: {urls}."
142
+ return msg
143
+
144
+ if datasets_dtype in pa.__dict__:
145
+ return pa.__dict__[datasets_dtype]()
146
+
147
+ if (datasets_dtype + "_") in pa.__dict__:
148
+ return pa.__dict__[datasets_dtype + "_"]()
149
+
150
+ timestamp_matches = re.search(r"^timestamp\[(.*)\]$", datasets_dtype)
151
+ if timestamp_matches:
152
+ timestamp_internals = timestamp_matches.group(1)
153
+ internals_matches = re.search(r"^(s|ms|us|ns),\s*tz=([a-zA-Z0-9/_+\-:]*)$", timestamp_internals)
154
+ if timestamp_internals in ["s", "ms", "us", "ns"]:
155
+ return pa.timestamp(timestamp_internals)
156
+ elif internals_matches:
157
+ return pa.timestamp(internals_matches.group(1), internals_matches.group(2))
158
+ else:
159
+ raise ValueError(
160
+ _dtype_error_msg(
161
+ datasets_dtype,
162
+ "timestamp",
163
+ examples=["timestamp[us]", "timestamp[us, tz=America/New_York"],
164
+ urls=["https://arrow.apache.org/docs/python/generated/pyarrow.timestamp.html"],
165
+ )
166
+ )
167
+
168
+ duration_matches = re.search(r"^duration\[(.*)\]$", datasets_dtype)
169
+ if duration_matches:
170
+ duration_internals = duration_matches.group(1)
171
+ if duration_internals in ["s", "ms", "us", "ns"]:
172
+ return pa.duration(duration_internals)
173
+ else:
174
+ raise ValueError(
175
+ _dtype_error_msg(
176
+ datasets_dtype,
177
+ "duration",
178
+ examples=["duration[s]", "duration[us]"],
179
+ urls=["https://arrow.apache.org/docs/python/generated/pyarrow.duration.html"],
180
+ )
181
+ )
182
+
183
+ time_matches = re.search(r"^time(.*)\[(.*)\]$", datasets_dtype)
184
+ if time_matches:
185
+ time_internals_bits = time_matches.group(1)
186
+ if time_internals_bits == "32":
187
+ time_internals_unit = time_matches.group(2)
188
+ if time_internals_unit in ["s", "ms"]:
189
+ return pa.time32(time_internals_unit)
190
+ else:
191
+ raise ValueError(
192
+ f"{time_internals_unit} is not a valid unit for the pyarrow time32 type. Supported units: s (second) and ms (millisecond)."
193
+ )
194
+ elif time_internals_bits == "64":
195
+ time_internals_unit = time_matches.group(2)
196
+ if time_internals_unit in ["us", "ns"]:
197
+ return pa.time64(time_internals_unit)
198
+ else:
199
+ raise ValueError(
200
+ f"{time_internals_unit} is not a valid unit for the pyarrow time64 type. Supported units: us (microsecond) and ns (nanosecond)."
201
+ )
202
+ else:
203
+ raise ValueError(
204
+ _dtype_error_msg(
205
+ datasets_dtype,
206
+ "time",
207
+ examples=["time32[s]", "time64[us]"],
208
+ urls=[
209
+ "https://arrow.apache.org/docs/python/generated/pyarrow.time32.html",
210
+ "https://arrow.apache.org/docs/python/generated/pyarrow.time64.html",
211
+ ],
212
+ )
213
+ )
214
+
215
+ decimal_matches = re.search(r"^decimal(.*)\((.*)\)$", datasets_dtype)
216
+ if decimal_matches:
217
+ decimal_internals_bits = decimal_matches.group(1)
218
+ if decimal_internals_bits == "128":
219
+ decimal_internals_precision_and_scale = re.search(r"^(\d+),\s*(-?\d+)$", decimal_matches.group(2))
220
+ if decimal_internals_precision_and_scale:
221
+ precision = decimal_internals_precision_and_scale.group(1)
222
+ scale = decimal_internals_precision_and_scale.group(2)
223
+ return pa.decimal128(int(precision), int(scale))
224
+ else:
225
+ raise ValueError(
226
+ _dtype_error_msg(
227
+ datasets_dtype,
228
+ "decimal128",
229
+ examples=["decimal128(10, 2)", "decimal128(4, -2)"],
230
+ urls=["https://arrow.apache.org/docs/python/generated/pyarrow.decimal128.html"],
231
+ )
232
+ )
233
+ elif decimal_internals_bits == "256":
234
+ decimal_internals_precision_and_scale = re.search(r"^(\d+),\s*(-?\d+)$", decimal_matches.group(2))
235
+ if decimal_internals_precision_and_scale:
236
+ precision = decimal_internals_precision_and_scale.group(1)
237
+ scale = decimal_internals_precision_and_scale.group(2)
238
+ return pa.decimal256(int(precision), int(scale))
239
+ else:
240
+ raise ValueError(
241
+ _dtype_error_msg(
242
+ datasets_dtype,
243
+ "decimal256",
244
+ examples=["decimal256(30, 2)", "decimal256(38, -4)"],
245
+ urls=["https://arrow.apache.org/docs/python/generated/pyarrow.decimal256.html"],
246
+ )
247
+ )
248
+ else:
249
+ raise ValueError(
250
+ _dtype_error_msg(
251
+ datasets_dtype,
252
+ "decimal",
253
+ examples=["decimal128(12, 3)", "decimal256(40, 6)"],
254
+ urls=[
255
+ "https://arrow.apache.org/docs/python/generated/pyarrow.decimal128.html",
256
+ "https://arrow.apache.org/docs/python/generated/pyarrow.decimal256.html",
257
+ ],
258
+ )
259
+ )
260
+
261
+ raise ValueError(
262
+ f"Neither {datasets_dtype} nor {datasets_dtype + '_'} seems to be a pyarrow data type. "
263
+ f"Please make sure to use a correct data type, see: "
264
+ f"https://arrow.apache.org/docs/python/api/datatypes.html#factory-functions"
265
+ )
266
+
267
+
268
+ def _cast_to_python_objects(obj: Any, only_1d_for_numpy: bool, optimize_list_casting: bool) -> tuple[Any, bool]:
269
+ """
270
+ Cast pytorch/tensorflow/pandas objects to python numpy array/lists.
271
+ It works recursively.
272
+
273
+ If `optimize_list_casting` is True, to avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be casted.
274
+ If the first element needs to be casted, then all the elements of the list will be casted, otherwise they'll stay the same.
275
+ This trick allows to cast objects that contain tokenizers outputs without iterating over every single token for example.
276
+
277
+ Args:
278
+ obj: the object (nested struct) to cast.
279
+ only_1d_for_numpy (bool): whether to keep the full multi-dim tensors as multi-dim numpy arrays, or convert them to
280
+ nested lists of 1-dimensional numpy arrays. This can be useful to keep only 1-d arrays to instantiate Arrow arrays.
281
+ Indeed Arrow only support converting 1-dimensional array values.
282
+ optimize_list_casting (bool): whether to optimize list casting by checking the first non-null element to see if it needs to be casted
283
+ and if it doesn't, not checking the rest of the list elements.
284
+
285
+ Returns:
286
+ casted_obj: the casted object
287
+ has_changed (bool): True if the object has been changed, False if it is identical
288
+ """
289
+
290
+ if config.TF_AVAILABLE and "tensorflow" in sys.modules:
291
+ import tensorflow as tf
292
+
293
+ if config.TORCH_AVAILABLE and "torch" in sys.modules:
294
+ import torch
295
+
296
+ if config.JAX_AVAILABLE and "jax" in sys.modules:
297
+ import jax.numpy as jnp
298
+
299
+ if config.PIL_AVAILABLE and "PIL" in sys.modules:
300
+ import PIL.Image
301
+
302
+ if config.PDFPLUMBER_AVAILABLE and "pdfplumber" in sys.modules:
303
+ import pdfplumber
304
+
305
+ if isinstance(obj, np.ndarray):
306
+ if obj.ndim == 0:
307
+ return obj[()], True
308
+ elif not only_1d_for_numpy or obj.ndim == 1:
309
+ return obj, False
310
+ else:
311
+ return (
312
+ [
313
+ _cast_to_python_objects(
314
+ x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
315
+ )[0]
316
+ for x in obj
317
+ ],
318
+ True,
319
+ )
320
+ elif config.TORCH_AVAILABLE and "torch" in sys.modules and isinstance(obj, torch.Tensor):
321
+ if obj.dtype == torch.bfloat16:
322
+ return _cast_to_python_objects(
323
+ obj.detach().to(torch.float).cpu().numpy(),
324
+ only_1d_for_numpy=only_1d_for_numpy,
325
+ optimize_list_casting=optimize_list_casting,
326
+ )[0], True
327
+ if obj.ndim == 0:
328
+ return obj.detach().cpu().numpy()[()], True
329
+ elif not only_1d_for_numpy or obj.ndim == 1:
330
+ return obj.detach().cpu().numpy(), True
331
+ else:
332
+ return (
333
+ [
334
+ _cast_to_python_objects(
335
+ x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
336
+ )[0]
337
+ for x in obj.detach().cpu().numpy()
338
+ ],
339
+ True,
340
+ )
341
+ elif config.TF_AVAILABLE and "tensorflow" in sys.modules and isinstance(obj, tf.Tensor):
342
+ if obj.ndim == 0:
343
+ return obj.numpy()[()], True
344
+ elif not only_1d_for_numpy or obj.ndim == 1:
345
+ return obj.numpy(), True
346
+ else:
347
+ return (
348
+ [
349
+ _cast_to_python_objects(
350
+ x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
351
+ )[0]
352
+ for x in obj.numpy()
353
+ ],
354
+ True,
355
+ )
356
+ elif config.JAX_AVAILABLE and "jax" in sys.modules and isinstance(obj, jnp.ndarray):
357
+ if obj.ndim == 0:
358
+ return np.asarray(obj)[()], True
359
+ elif not only_1d_for_numpy or obj.ndim == 1:
360
+ return np.asarray(obj), True
361
+ else:
362
+ return (
363
+ [
364
+ _cast_to_python_objects(
365
+ x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
366
+ )[0]
367
+ for x in np.asarray(obj)
368
+ ],
369
+ True,
370
+ )
371
+ elif config.PIL_AVAILABLE and "PIL" in sys.modules and isinstance(obj, PIL.Image.Image):
372
+ return encode_pil_image(obj), True
373
+ elif config.PDFPLUMBER_AVAILABLE and "pdfplumber" in sys.modules and isinstance(obj, pdfplumber.pdf.PDF):
374
+ return encode_pdfplumber_pdf(obj), True
375
+ elif isinstance(obj, pd.Series):
376
+ return (
377
+ _cast_to_python_objects(
378
+ obj.tolist(), only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
379
+ )[0],
380
+ True,
381
+ )
382
+ elif isinstance(obj, pd.DataFrame):
383
+ return (
384
+ {
385
+ key: _cast_to_python_objects(
386
+ value, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
387
+ )[0]
388
+ for key, value in obj.to_dict("series").items()
389
+ },
390
+ True,
391
+ )
392
+ elif isinstance(obj, pd.Timestamp):
393
+ return obj.to_pydatetime(), True
394
+ elif isinstance(obj, pd.Timedelta):
395
+ return obj.to_pytimedelta(), True
396
+ elif isinstance(obj, Mapping):
397
+ has_changed = not isinstance(obj, dict)
398
+ output = {}
399
+ for k, v in obj.items():
400
+ casted_v, has_changed_v = _cast_to_python_objects(
401
+ v, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
402
+ )
403
+ has_changed |= has_changed_v
404
+ output[k] = casted_v
405
+ return output if has_changed else obj, has_changed
406
+ elif hasattr(obj, "__array__"):
407
+ if np.isscalar(obj):
408
+ return obj, False
409
+ else:
410
+ return (
411
+ _cast_to_python_objects(
412
+ obj.__array__(), only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
413
+ )[0],
414
+ True,
415
+ )
416
+ elif isinstance(obj, (list, tuple)):
417
+ if len(obj) > 0:
418
+ for first_elmt in obj:
419
+ if _check_non_null_non_empty_recursive(first_elmt):
420
+ break
421
+ casted_first_elmt, has_changed_first_elmt = _cast_to_python_objects(
422
+ first_elmt, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
423
+ )
424
+ if has_changed_first_elmt or not optimize_list_casting:
425
+ return (
426
+ [
427
+ _cast_to_python_objects(
428
+ elmt, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
429
+ )[0]
430
+ for elmt in obj
431
+ ],
432
+ True,
433
+ )
434
+ else:
435
+ if isinstance(obj, (list, tuple)):
436
+ return obj, False
437
+ else:
438
+ return list(obj), True
439
+ else:
440
+ return obj, False
441
+ else:
442
+ return obj, False
443
+
444
+
445
+ def cast_to_python_objects(obj: Any, only_1d_for_numpy=False, optimize_list_casting=True) -> Any:
446
+ """
447
+ Cast numpy/pytorch/tensorflow/pandas objects to python lists.
448
+ It works recursively.
449
+
450
+ If `optimize_list_casting` is True, To avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be casted.
451
+ If the first element needs to be casted, then all the elements of the list will be casted, otherwise they'll stay the same.
452
+ This trick allows to cast objects that contain tokenizers outputs without iterating over every single token for example.
453
+
454
+ Args:
455
+ obj: the object (nested struct) to cast
456
+ only_1d_for_numpy (bool, default ``False``): whether to keep the full multi-dim tensors as multi-dim numpy arrays, or convert them to
457
+ nested lists of 1-dimensional numpy arrays. This can be useful to keep only 1-d arrays to instantiate Arrow arrays.
458
+ Indeed Arrow only support converting 1-dimensional array values.
459
+ optimize_list_casting (bool, default ``True``): whether to optimize list casting by checking the first non-null element to see if it needs to be casted
460
+ and if it doesn't, not checking the rest of the list elements.
461
+
462
+ Returns:
463
+ casted_obj: the casted object
464
+ """
465
+ return _cast_to_python_objects(
466
+ obj, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
467
+ )[0]
468
+
469
+
470
+ @dataclass
471
+ class Value:
472
+ """
473
+ Scalar feature value of a particular data type.
474
+
475
+ The possible dtypes of `Value` are as follows:
476
+ - `null`
477
+ - `bool`
478
+ - `int8`
479
+ - `int16`
480
+ - `int32`
481
+ - `int64`
482
+ - `uint8`
483
+ - `uint16`
484
+ - `uint32`
485
+ - `uint64`
486
+ - `float16`
487
+ - `float32` (alias float)
488
+ - `float64` (alias double)
489
+ - `time32[(s|ms)]`
490
+ - `time64[(us|ns)]`
491
+ - `timestamp[(s|ms|us|ns)]`
492
+ - `timestamp[(s|ms|us|ns), tz=(tzstring)]`
493
+ - `date32`
494
+ - `date64`
495
+ - `duration[(s|ms|us|ns)]`
496
+ - `decimal128(precision, scale)`
497
+ - `decimal256(precision, scale)`
498
+ - `binary`
499
+ - `large_binary`
500
+ - `string`
501
+ - `large_string`
502
+
503
+ Args:
504
+ dtype (`str`):
505
+ Name of the data type.
506
+
507
+ Example:
508
+
509
+ ```py
510
+ >>> from datasets import Features
511
+ >>> features = Features({'stars': Value(dtype='int32')})
512
+ >>> features
513
+ {'stars': Value(dtype='int32', id=None)}
514
+ ```
515
+ """
516
+
517
+ dtype: str
518
+ id: Optional[str] = None
519
+ # Automatically constructed
520
+ pa_type: ClassVar[Any] = None
521
+ _type: str = field(default="Value", init=False, repr=False)
522
+
523
+ def __post_init__(self):
524
+ if self.dtype == "double": # fix inferred type
525
+ self.dtype = "float64"
526
+ if self.dtype == "float": # fix inferred type
527
+ self.dtype = "float32"
528
+ self.pa_type = string_to_arrow(self.dtype)
529
+
530
+ def __call__(self):
531
+ return self.pa_type
532
+
533
+ def encode_example(self, value):
534
+ if pa.types.is_boolean(self.pa_type):
535
+ return bool(value)
536
+ elif pa.types.is_integer(self.pa_type):
537
+ return int(value)
538
+ elif pa.types.is_floating(self.pa_type):
539
+ return float(value)
540
+ elif pa.types.is_string(self.pa_type):
541
+ return str(value)
542
+ else:
543
+ return value
544
+
545
+
546
+ class _ArrayXD:
547
+ def __post_init__(self):
548
+ self.shape = tuple(self.shape)
549
+
550
+ def __call__(self):
551
+ pa_type = globals()[self.__class__.__name__ + "ExtensionType"](self.shape, self.dtype)
552
+ return pa_type
553
+
554
+ def encode_example(self, value):
555
+ return value
556
+
557
+
558
+ @dataclass
559
+ class Array2D(_ArrayXD):
560
+ """Create a two-dimensional array.
561
+
562
+ Args:
563
+ shape (`tuple`):
564
+ Size of each dimension.
565
+ dtype (`str`):
566
+ Name of the data type.
567
+
568
+ Example:
569
+
570
+ ```py
571
+ >>> from datasets import Features
572
+ >>> features = Features({'x': Array2D(shape=(1, 3), dtype='int32')})
573
+ ```
574
+ """
575
+
576
+ shape: tuple
577
+ dtype: str
578
+ id: Optional[str] = None
579
+ # Automatically constructed
580
+ _type: str = field(default="Array2D", init=False, repr=False)
581
+
582
+
583
+ @dataclass
584
+ class Array3D(_ArrayXD):
585
+ """Create a three-dimensional array.
586
+
587
+ Args:
588
+ shape (`tuple`):
589
+ Size of each dimension.
590
+ dtype (`str`):
591
+ Name of the data type.
592
+
593
+ Example:
594
+
595
+ ```py
596
+ >>> from datasets import Features
597
+ >>> features = Features({'x': Array3D(shape=(1, 2, 3), dtype='int32')})
598
+ ```
599
+ """
600
+
601
+ shape: tuple
602
+ dtype: str
603
+ id: Optional[str] = None
604
+ # Automatically constructed
605
+ _type: str = field(default="Array3D", init=False, repr=False)
606
+
607
+
608
+ @dataclass
609
+ class Array4D(_ArrayXD):
610
+ """Create a four-dimensional array.
611
+
612
+ Args:
613
+ shape (`tuple`):
614
+ Size of each dimension.
615
+ dtype (`str`):
616
+ Name of the data type.
617
+
618
+ Example:
619
+
620
+ ```py
621
+ >>> from datasets import Features
622
+ >>> features = Features({'x': Array4D(shape=(1, 2, 2, 3), dtype='int32')})
623
+ ```
624
+ """
625
+
626
+ shape: tuple
627
+ dtype: str
628
+ id: Optional[str] = None
629
+ # Automatically constructed
630
+ _type: str = field(default="Array4D", init=False, repr=False)
631
+
632
+
633
+ @dataclass
634
+ class Array5D(_ArrayXD):
635
+ """Create a five-dimensional array.
636
+
637
+ Args:
638
+ shape (`tuple`):
639
+ Size of each dimension.
640
+ dtype (`str`):
641
+ Name of the data type.
642
+
643
+ Example:
644
+
645
+ ```py
646
+ >>> from datasets import Features
647
+ >>> features = Features({'x': Array5D(shape=(1, 2, 2, 3, 3), dtype='int32')})
648
+ ```
649
+ """
650
+
651
+ shape: tuple
652
+ dtype: str
653
+ id: Optional[str] = None
654
+ # Automatically constructed
655
+ _type: str = field(default="Array5D", init=False, repr=False)
656
+
657
+
658
+ class _ArrayXDExtensionType(pa.ExtensionType):
659
+ ndims: Optional[int] = None
660
+
661
+ def __init__(self, shape: tuple, dtype: str):
662
+ if self.ndims is None or self.ndims <= 1:
663
+ raise ValueError("You must instantiate an array type with a value for dim that is > 1")
664
+ if len(shape) != self.ndims:
665
+ raise ValueError(f"shape={shape} and ndims={self.ndims} don't match")
666
+ for dim in range(1, self.ndims):
667
+ if shape[dim] is None:
668
+ raise ValueError(f"Support only dynamic size on first dimension. Got: {shape}")
669
+ self.shape = tuple(shape)
670
+ self.value_type = dtype
671
+ self.storage_dtype = self._generate_dtype(self.value_type)
672
+ pa.ExtensionType.__init__(self, self.storage_dtype, f"{self.__class__.__module__}.{self.__class__.__name__}")
673
+
674
+ def __arrow_ext_serialize__(self):
675
+ return json.dumps((self.shape, self.value_type)).encode()
676
+
677
+ @classmethod
678
+ def __arrow_ext_deserialize__(cls, storage_type, serialized):
679
+ args = json.loads(serialized)
680
+ return cls(*args)
681
+
682
+ # This was added to pa.ExtensionType in pyarrow >= 13.0.0
683
+ def __reduce__(self):
684
+ return self.__arrow_ext_deserialize__, (self.storage_type, self.__arrow_ext_serialize__())
685
+
686
+ def __hash__(self):
687
+ return hash((self.__class__, self.shape, self.value_type))
688
+
689
+ def __arrow_ext_class__(self):
690
+ return ArrayExtensionArray
691
+
692
+ def _generate_dtype(self, dtype):
693
+ dtype = string_to_arrow(dtype)
694
+ for d in reversed(self.shape):
695
+ dtype = pa.list_(dtype)
696
+ # Don't specify the size of the list, since fixed length list arrays have issues
697
+ # being validated after slicing in pyarrow 0.17.1
698
+ return dtype
699
+
700
+ def to_pandas_dtype(self):
701
+ return PandasArrayExtensionDtype(self.value_type)
702
+
703
+
704
+ class Array2DExtensionType(_ArrayXDExtensionType):
705
+ ndims = 2
706
+
707
+
708
+ class Array3DExtensionType(_ArrayXDExtensionType):
709
+ ndims = 3
710
+
711
+
712
+ class Array4DExtensionType(_ArrayXDExtensionType):
713
+ ndims = 4
714
+
715
+
716
+ class Array5DExtensionType(_ArrayXDExtensionType):
717
+ ndims = 5
718
+
719
+
720
+ # Register the extension types for deserialization
721
+ pa.register_extension_type(Array2DExtensionType((1, 2), "int64"))
722
+ pa.register_extension_type(Array3DExtensionType((1, 2, 3), "int64"))
723
+ pa.register_extension_type(Array4DExtensionType((1, 2, 3, 4), "int64"))
724
+ pa.register_extension_type(Array5DExtensionType((1, 2, 3, 4, 5), "int64"))
725
+
726
+
727
+ def _is_zero_copy_only(pa_type: pa.DataType, unnest: bool = False) -> bool:
728
+ """
729
+ When converting a pyarrow array to a numpy array, we must know whether this could be done in zero-copy or not.
730
+ This function returns the value of the ``zero_copy_only`` parameter to pass to ``.to_numpy()``, given the type of the pyarrow array.
731
+
732
+ # zero copy is available for all primitive types except booleans and temporal types (date, time, timestamp or duration)
733
+ # primitive types are types for which the physical representation in arrow and in numpy
734
+ # https://github.com/wesm/arrow/blob/c07b9b48cf3e0bbbab493992a492ae47e5b04cad/python/pyarrow/types.pxi#L821
735
+ # see https://arrow.apache.org/docs/python/generated/pyarrow.Array.html#pyarrow.Array.to_numpy
736
+ # and https://issues.apache.org/jira/browse/ARROW-2871?jql=text%20~%20%22boolean%20to_numpy%22
737
+ """
738
+
739
+ def _unnest_pa_type(pa_type: pa.DataType) -> pa.DataType:
740
+ if pa.types.is_list(pa_type):
741
+ return _unnest_pa_type(pa_type.value_type)
742
+ return pa_type
743
+
744
+ if unnest:
745
+ pa_type = _unnest_pa_type(pa_type)
746
+ return pa.types.is_primitive(pa_type) and not (pa.types.is_boolean(pa_type) or pa.types.is_temporal(pa_type))
747
+
748
+
749
+ class ArrayExtensionArray(pa.ExtensionArray):
750
+ def __array__(self):
751
+ zero_copy_only = _is_zero_copy_only(self.storage.type, unnest=True)
752
+ return self.to_numpy(zero_copy_only=zero_copy_only)
753
+
754
+ def __getitem__(self, i):
755
+ return self.storage[i]
756
+
757
+ def to_numpy(self, zero_copy_only=True):
758
+ storage: pa.ListArray = self.storage
759
+ null_mask = storage.is_null().to_numpy(zero_copy_only=False)
760
+
761
+ if self.type.shape[0] is not None:
762
+ size = 1
763
+ null_indices = np.arange(len(storage))[null_mask] - np.arange(np.sum(null_mask))
764
+
765
+ for i in range(self.type.ndims):
766
+ size *= self.type.shape[i]
767
+ storage = storage.flatten()
768
+ numpy_arr = storage.to_numpy(zero_copy_only=zero_copy_only)
769
+ numpy_arr = numpy_arr.reshape(len(self) - len(null_indices), *self.type.shape)
770
+
771
+ if len(null_indices):
772
+ numpy_arr = np.insert(numpy_arr.astype(np.float64), null_indices, np.nan, axis=0)
773
+
774
+ else:
775
+ shape = self.type.shape
776
+ ndims = self.type.ndims
777
+ arrays = []
778
+ first_dim_offsets = np.array([off.as_py() for off in storage.offsets])
779
+ for i, is_null in enumerate(null_mask):
780
+ if is_null:
781
+ arrays.append(np.nan)
782
+ else:
783
+ storage_el = storage[i : i + 1]
784
+ first_dim = first_dim_offsets[i + 1] - first_dim_offsets[i]
785
+ # flatten storage
786
+ for _ in range(ndims):
787
+ storage_el = storage_el.flatten()
788
+
789
+ numpy_arr = storage_el.to_numpy(zero_copy_only=zero_copy_only)
790
+ arrays.append(numpy_arr.reshape(first_dim, *shape[1:]))
791
+
792
+ if len(np.unique(np.diff(first_dim_offsets))) > 1:
793
+ # ragged
794
+ numpy_arr = np.empty(len(arrays), dtype=object)
795
+ numpy_arr[:] = arrays
796
+ else:
797
+ numpy_arr = np.array(arrays)
798
+
799
+ return numpy_arr
800
+
801
+ def to_pylist(self, maps_as_pydicts: Optional[Literal["lossy", "strict"]] = None):
802
+ zero_copy_only = _is_zero_copy_only(self.storage.type, unnest=True)
803
+ numpy_arr = self.to_numpy(zero_copy_only=zero_copy_only)
804
+ if self.type.shape[0] is None and numpy_arr.dtype == object:
805
+ return [arr.tolist() for arr in numpy_arr.tolist()]
806
+ else:
807
+ return numpy_arr.tolist()
808
+
809
+
810
+ class PandasArrayExtensionDtype(PandasExtensionDtype):
811
+ _metadata = "value_type"
812
+
813
+ def __init__(self, value_type: Union["PandasArrayExtensionDtype", np.dtype]):
814
+ self._value_type = value_type
815
+
816
+ def __from_arrow__(self, array: Union[pa.Array, pa.ChunkedArray]):
817
+ if isinstance(array, pa.ChunkedArray):
818
+ array = array.type.wrap_array(pa.concat_arrays([chunk.storage for chunk in array.chunks]))
819
+ zero_copy_only = _is_zero_copy_only(array.storage.type, unnest=True)
820
+ numpy_arr = array.to_numpy(zero_copy_only=zero_copy_only)
821
+ return PandasArrayExtensionArray(numpy_arr)
822
+
823
+ @classmethod
824
+ def construct_array_type(cls):
825
+ return PandasArrayExtensionArray
826
+
827
+ @property
828
+ def type(self) -> type:
829
+ return np.ndarray
830
+
831
+ @property
832
+ def kind(self) -> str:
833
+ return "O"
834
+
835
+ @property
836
+ def name(self) -> str:
837
+ return f"array[{self.value_type}]"
838
+
839
+ @property
840
+ def value_type(self) -> np.dtype:
841
+ return self._value_type
842
+
843
+
844
+ class PandasArrayExtensionArray(PandasExtensionArray):
845
+ def __init__(self, data: np.ndarray, copy: bool = False):
846
+ self._data = data if not copy else np.array(data)
847
+ self._dtype = PandasArrayExtensionDtype(data.dtype)
848
+
849
+ def __array__(self, dtype=None):
850
+ """
851
+ Convert to NumPy Array.
852
+ Note that Pandas expects a 1D array when dtype is set to object.
853
+ But for other dtypes, the returned shape is the same as the one of ``data``.
854
+
855
+ More info about pandas 1D requirement for PandasExtensionArray here:
856
+ https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.api.extensions.ExtensionArray.html#pandas.api.extensions.ExtensionArray
857
+
858
+ """
859
+ if dtype == np.dtype(object):
860
+ out = np.empty(len(self._data), dtype=object)
861
+ for i in range(len(self._data)):
862
+ out[i] = self._data[i]
863
+ return out
864
+ if dtype is None:
865
+ return self._data
866
+ else:
867
+ return self._data.astype(dtype)
868
+
869
+ def copy(self, deep: bool = False) -> "PandasArrayExtensionArray":
870
+ return PandasArrayExtensionArray(self._data, copy=True)
871
+
872
+ @classmethod
873
+ def _from_sequence(
874
+ cls, scalars, dtype: Optional[PandasArrayExtensionDtype] = None, copy: bool = False
875
+ ) -> "PandasArrayExtensionArray":
876
+ if len(scalars) > 1 and all(
877
+ isinstance(x, np.ndarray) and x.shape == scalars[0].shape and x.dtype == scalars[0].dtype for x in scalars
878
+ ):
879
+ data = np.array(scalars, dtype=dtype if dtype is None else dtype.value_type, copy=copy)
880
+ else:
881
+ data = np.empty(len(scalars), dtype=object)
882
+ data[:] = scalars
883
+ return cls(data, copy=copy)
884
+
885
+ @classmethod
886
+ def _concat_same_type(cls, to_concat: Sequence_["PandasArrayExtensionArray"]) -> "PandasArrayExtensionArray":
887
+ if len(to_concat) > 1 and all(
888
+ va._data.shape == to_concat[0]._data.shape and va._data.dtype == to_concat[0]._data.dtype
889
+ for va in to_concat
890
+ ):
891
+ data = np.vstack([va._data for va in to_concat])
892
+ else:
893
+ data = np.empty(len(to_concat), dtype=object)
894
+ data[:] = [va._data for va in to_concat]
895
+ return cls(data, copy=False)
896
+
897
+ @property
898
+ def dtype(self) -> PandasArrayExtensionDtype:
899
+ return self._dtype
900
+
901
+ @property
902
+ def nbytes(self) -> int:
903
+ return self._data.nbytes
904
+
905
+ def isna(self) -> np.ndarray:
906
+ return np.array([pd.isna(arr).any() for arr in self._data])
907
+
908
+ def __setitem__(self, key: Union[int, slice, np.ndarray], value: Any) -> None:
909
+ raise NotImplementedError()
910
+
911
+ def __getitem__(self, item: Union[int, slice, np.ndarray]) -> Union[np.ndarray, "PandasArrayExtensionArray"]:
912
+ if isinstance(item, int):
913
+ return self._data[item]
914
+ return PandasArrayExtensionArray(self._data[item], copy=False)
915
+
916
+ def take(
917
+ self, indices: Sequence_[int], allow_fill: bool = False, fill_value: bool = None
918
+ ) -> "PandasArrayExtensionArray":
919
+ indices: np.ndarray = np.asarray(indices, dtype=int)
920
+ if allow_fill:
921
+ fill_value = (
922
+ self.dtype.na_value if fill_value is None else np.asarray(fill_value, dtype=self.dtype.value_type)
923
+ )
924
+ mask = indices == -1
925
+ if (indices < -1).any():
926
+ raise ValueError("Invalid value in `indices`, must be all >= -1 for `allow_fill` is True")
927
+ elif len(self) > 0:
928
+ pass
929
+ elif not np.all(mask):
930
+ raise IndexError("Invalid take for empty PandasArrayExtensionArray, must be all -1.")
931
+ else:
932
+ data = np.array([fill_value] * len(indices), dtype=self.dtype.value_type)
933
+ return PandasArrayExtensionArray(data, copy=False)
934
+ took = self._data.take(indices, axis=0)
935
+ if allow_fill and mask.any():
936
+ took[mask] = [fill_value] * np.sum(mask)
937
+ return PandasArrayExtensionArray(took, copy=False)
938
+
939
+ def __len__(self) -> int:
940
+ return len(self._data)
941
+
942
+ def __eq__(self, other) -> np.ndarray:
943
+ if not isinstance(other, PandasArrayExtensionArray):
944
+ raise NotImplementedError(f"Invalid type to compare to: {type(other)}")
945
+ return (self._data == other._data).all()
946
+
947
+
948
+ def pandas_types_mapper(dtype):
949
+ if isinstance(dtype, _ArrayXDExtensionType):
950
+ return PandasArrayExtensionDtype(dtype.value_type)
951
+
952
+
953
+ @dataclass
954
+ class ClassLabel:
955
+ """Feature type for integer class labels.
956
+
957
+ There are 3 ways to define a `ClassLabel`, which correspond to the 3 arguments:
958
+
959
+ * `num_classes`: Create 0 to (num_classes-1) labels.
960
+ * `names`: List of label strings.
961
+ * `names_file`: File containing the list of labels.
962
+
963
+ Under the hood the labels are stored as integers.
964
+ You can use negative integers to represent unknown/missing labels.
965
+
966
+ Args:
967
+ num_classes (`int`, *optional*):
968
+ Number of classes. All labels must be < `num_classes`.
969
+ names (`list` of `str`, *optional*):
970
+ String names for the integer classes.
971
+ The order in which the names are provided is kept.
972
+ names_file (`str`, *optional*):
973
+ Path to a file with names for the integer classes, one per line.
974
+
975
+ Example:
976
+
977
+ ```py
978
+ >>> from datasets import Features, ClassLabel
979
+ >>> features = Features({'label': ClassLabel(num_classes=3, names=['bad', 'ok', 'good'])})
980
+ >>> features
981
+ {'label': ClassLabel(names=['bad', 'ok', 'good'], id=None)}
982
+ ```
983
+ """
984
+
985
+ num_classes: InitVar[Optional[int]] = None # Pseudo-field: ignored by asdict/fields when converting to/from dict
986
+ names: list[str] = None
987
+ names_file: InitVar[Optional[str]] = None # Pseudo-field: ignored by asdict/fields when converting to/from dict
988
+ id: Optional[str] = None
989
+ # Automatically constructed
990
+ dtype: ClassVar[str] = "int64"
991
+ pa_type: ClassVar[Any] = pa.int64()
992
+ _str2int: ClassVar[dict[str, int]] = None
993
+ _int2str: ClassVar[dict[int, int]] = None
994
+ _type: str = field(default="ClassLabel", init=False, repr=False)
995
+
996
+ def __post_init__(self, num_classes, names_file):
997
+ self.num_classes = num_classes
998
+ self.names_file = names_file
999
+ if self.names_file is not None and self.names is not None:
1000
+ raise ValueError("Please provide either names or names_file but not both.")
1001
+ # Set self.names
1002
+ if self.names is None:
1003
+ if self.names_file is not None:
1004
+ self.names = self._load_names_from_file(self.names_file)
1005
+ elif self.num_classes is not None:
1006
+ self.names = [str(i) for i in range(self.num_classes)]
1007
+ else:
1008
+ raise ValueError("Please provide either num_classes, names or names_file.")
1009
+ elif not isinstance(self.names, SequenceABC):
1010
+ raise TypeError(f"Please provide names as a list, is {type(self.names)}")
1011
+ # Set self.num_classes
1012
+ if self.num_classes is None:
1013
+ self.num_classes = len(self.names)
1014
+ elif self.num_classes != len(self.names):
1015
+ raise ValueError(
1016
+ "ClassLabel number of names do not match the defined num_classes. "
1017
+ f"Got {len(self.names)} names VS {self.num_classes} num_classes"
1018
+ )
1019
+ # Prepare mappings
1020
+ self._int2str = [str(name) for name in self.names]
1021
+ self._str2int = {name: i for i, name in enumerate(self._int2str)}
1022
+ if len(self._int2str) != len(self._str2int):
1023
+ raise ValueError("Some label names are duplicated. Each label name should be unique.")
1024
+
1025
+ def __call__(self):
1026
+ return self.pa_type
1027
+
1028
+ def str2int(self, values: Union[str, Iterable]) -> Union[int, Iterable]:
1029
+ """Conversion class name `string` => `integer`.
1030
+
1031
+ Example:
1032
+
1033
+ ```py
1034
+ >>> from datasets import load_dataset
1035
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
1036
+ >>> ds.features["label"].str2int('neg')
1037
+ 0
1038
+ ```
1039
+ """
1040
+ if not isinstance(values, str) and not isinstance(values, Iterable):
1041
+ raise ValueError(
1042
+ f"Values {values} should be a string or an Iterable (list, numpy array, pytorch, tensorflow tensors)"
1043
+ )
1044
+ return_list = True
1045
+ if isinstance(values, str):
1046
+ values = [values]
1047
+ return_list = False
1048
+
1049
+ output = [self._strval2int(value) for value in values]
1050
+ return output if return_list else output[0]
1051
+
1052
+ def _strval2int(self, value: str) -> int:
1053
+ failed_parse = False
1054
+ value = str(value)
1055
+ # first attempt - raw string value
1056
+ int_value = self._str2int.get(value)
1057
+ if int_value is None:
1058
+ # second attempt - strip whitespace
1059
+ int_value = self._str2int.get(value.strip())
1060
+ if int_value is None:
1061
+ # third attempt - convert str to int
1062
+ try:
1063
+ int_value = int(value)
1064
+ except ValueError:
1065
+ failed_parse = True
1066
+ else:
1067
+ if int_value < -1 or int_value >= self.num_classes:
1068
+ failed_parse = True
1069
+ if failed_parse:
1070
+ raise ValueError(f"Invalid string class label {value}")
1071
+ return int_value
1072
+
1073
+ def int2str(self, values: Union[int, Iterable]) -> Union[str, Iterable]:
1074
+ """Conversion `integer` => class name `string`.
1075
+
1076
+ Regarding unknown/missing labels: passing negative integers raises `ValueError`.
1077
+
1078
+ Example:
1079
+
1080
+ ```py
1081
+ >>> from datasets import load_dataset
1082
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
1083
+ >>> ds.features["label"].int2str(0)
1084
+ 'neg'
1085
+ ```
1086
+ """
1087
+ if not isinstance(values, int) and not isinstance(values, Iterable):
1088
+ raise ValueError(
1089
+ f"Values {values} should be an integer or an Iterable (list, numpy array, pytorch, tensorflow tensors)"
1090
+ )
1091
+ return_list = True
1092
+ if isinstance(values, int):
1093
+ values = [values]
1094
+ return_list = False
1095
+
1096
+ for v in values:
1097
+ if not 0 <= v < self.num_classes:
1098
+ raise ValueError(f"Invalid integer class label {v:d}")
1099
+
1100
+ output = [self._int2str[int(v)] for v in values]
1101
+ return output if return_list else output[0]
1102
+
1103
+ def encode_example(self, example_data):
1104
+ if self.num_classes is None:
1105
+ raise ValueError(
1106
+ "Trying to use ClassLabel feature with undefined number of class. "
1107
+ "Please set ClassLabel.names or num_classes."
1108
+ )
1109
+
1110
+ # If a string is given, convert to associated integer
1111
+ if isinstance(example_data, str):
1112
+ example_data = self.str2int(example_data)
1113
+
1114
+ # Allowing -1 to mean no label.
1115
+ if not -1 <= example_data < self.num_classes:
1116
+ raise ValueError(f"Class label {example_data:d} greater than configured num_classes {self.num_classes}")
1117
+ return example_data
1118
+
1119
+ def cast_storage(self, storage: Union[pa.StringArray, pa.IntegerArray]) -> pa.Int64Array:
1120
+ """Cast an Arrow array to the `ClassLabel` arrow storage type.
1121
+ The Arrow types that can be converted to the `ClassLabel` pyarrow storage type are:
1122
+
1123
+ - `pa.string()`
1124
+ - `pa.int()`
1125
+
1126
+ Args:
1127
+ storage (`Union[pa.StringArray, pa.IntegerArray]`):
1128
+ PyArrow array to cast.
1129
+
1130
+ Returns:
1131
+ `pa.Int64Array`: Array in the `ClassLabel` arrow storage type.
1132
+ """
1133
+ if isinstance(storage, pa.IntegerArray) and len(storage) > 0:
1134
+ min_max = pc.min_max(storage).as_py()
1135
+ if min_max["max"] is not None and min_max["max"] >= self.num_classes:
1136
+ raise ValueError(
1137
+ f"Class label {min_max['max']} greater than configured num_classes {self.num_classes}"
1138
+ )
1139
+ elif isinstance(storage, pa.StringArray):
1140
+ storage = pa.array(
1141
+ [self._strval2int(label) if label is not None else None for label in storage.to_pylist()]
1142
+ )
1143
+ return array_cast(storage, self.pa_type)
1144
+
1145
+ @staticmethod
1146
+ def _load_names_from_file(names_filepath):
1147
+ with open(names_filepath, encoding="utf-8") as f:
1148
+ return [name.strip() for name in f.read().split("\n") if name.strip()] # Filter empty names
1149
+
1150
+
1151
+ @dataclass
1152
+ class Sequence:
1153
+ """Construct a list of feature from a single type or a dict of types.
1154
+ Mostly here for compatiblity with tfds.
1155
+
1156
+ Args:
1157
+ feature ([`FeatureType`]):
1158
+ A list of features of a single type or a dictionary of types.
1159
+ length (`int`):
1160
+ Length of the sequence.
1161
+
1162
+ Example:
1163
+
1164
+ ```py
1165
+ >>> from datasets import Features, Sequence, Value, ClassLabel
1166
+ >>> features = Features({'post': Sequence(feature={'text': Value(dtype='string'), 'upvotes': Value(dtype='int32'), 'label': ClassLabel(num_classes=2, names=['hot', 'cold'])})})
1167
+ >>> features
1168
+ {'post': Sequence(feature={'text': Value(dtype='string', id=None), 'upvotes': Value(dtype='int32', id=None), 'label': ClassLabel(names=['hot', 'cold'], id=None)}, length=-1, id=None)}
1169
+ ```
1170
+ """
1171
+
1172
+ feature: Any
1173
+ length: int = -1
1174
+ id: Optional[str] = None
1175
+ # Automatically constructed
1176
+ dtype: ClassVar[str] = "list"
1177
+ pa_type: ClassVar[Any] = None
1178
+ _type: str = field(default="Sequence", init=False, repr=False)
1179
+
1180
+
1181
+ @dataclass
1182
+ class LargeList:
1183
+ """Feature type for large list data composed of child feature data type.
1184
+
1185
+ It is backed by `pyarrow.LargeListType`, which is like `pyarrow.ListType` but with 64-bit rather than 32-bit offsets.
1186
+
1187
+ Args:
1188
+ feature ([`FeatureType`]):
1189
+ Child feature data type of each item within the large list.
1190
+ """
1191
+
1192
+ feature: Any
1193
+ id: Optional[str] = None
1194
+ # Automatically constructed
1195
+ pa_type: ClassVar[Any] = None
1196
+ _type: str = field(default="LargeList", init=False, repr=False)
1197
+
1198
+
1199
+ FeatureType = Union[
1200
+ dict,
1201
+ list,
1202
+ tuple,
1203
+ Value,
1204
+ ClassLabel,
1205
+ Translation,
1206
+ TranslationVariableLanguages,
1207
+ LargeList,
1208
+ Sequence,
1209
+ Array2D,
1210
+ Array3D,
1211
+ Array4D,
1212
+ Array5D,
1213
+ Audio,
1214
+ Image,
1215
+ Video,
1216
+ Pdf,
1217
+ ]
1218
+
1219
+
1220
+ def _check_non_null_non_empty_recursive(obj, schema: Optional[FeatureType] = None) -> bool:
1221
+ """
1222
+ Check if the object is not None.
1223
+ If the object is a list or a tuple, recursively check the first element of the sequence and stop if at any point the first element is not a sequence or is an empty sequence.
1224
+ """
1225
+ if obj is None:
1226
+ return False
1227
+ elif isinstance(obj, (list, tuple)) and (schema is None or isinstance(schema, (list, tuple, LargeList, Sequence))):
1228
+ if len(obj) > 0:
1229
+ if schema is None:
1230
+ pass
1231
+ elif isinstance(schema, (list, tuple)):
1232
+ schema = schema[0]
1233
+ else:
1234
+ schema = schema.feature
1235
+ return _check_non_null_non_empty_recursive(obj[0], schema)
1236
+ else:
1237
+ return False
1238
+ else:
1239
+ return True
1240
+
1241
+
1242
+ def get_nested_type(schema: FeatureType) -> pa.DataType:
1243
+ """
1244
+ get_nested_type() converts a datasets.FeatureType into a pyarrow.DataType, and acts as the inverse of
1245
+ generate_from_arrow_type().
1246
+
1247
+ It performs double-duty as the implementation of Features.type and handles the conversion of
1248
+ datasets.Feature->pa.struct
1249
+ """
1250
+ # Nested structures: we allow dict, list/tuples, sequences
1251
+ if isinstance(schema, Features):
1252
+ return pa.struct(
1253
+ {key: get_nested_type(schema[key]) for key in schema}
1254
+ ) # Features is subclass of dict, and dict order is deterministic since Python 3.6
1255
+ elif isinstance(schema, dict):
1256
+ return pa.struct(
1257
+ {key: get_nested_type(schema[key]) for key in schema}
1258
+ ) # however don't sort on struct types since the order matters
1259
+ elif isinstance(schema, (list, tuple)):
1260
+ if len(schema) != 1:
1261
+ raise ValueError("When defining list feature, you should just provide one example of the inner type")
1262
+ value_type = get_nested_type(schema[0])
1263
+ return pa.list_(value_type)
1264
+ elif isinstance(schema, LargeList):
1265
+ value_type = get_nested_type(schema.feature)
1266
+ return pa.large_list(value_type)
1267
+ elif isinstance(schema, Sequence):
1268
+ value_type = get_nested_type(schema.feature)
1269
+ # We allow to reverse list of dict => dict of list for compatibility with tfds
1270
+ if isinstance(schema.feature, dict):
1271
+ data_type = pa.struct({f.name: pa.list_(f.type, schema.length) for f in value_type})
1272
+ else:
1273
+ data_type = pa.list_(value_type, schema.length)
1274
+ return data_type
1275
+
1276
+ # Other objects are callable which returns their data type (ClassLabel, Array2D, Translation, Arrow datatype creation methods)
1277
+ return schema()
1278
+
1279
+
1280
+ def encode_nested_example(schema, obj, level=0):
1281
+ """Encode a nested example.
1282
+ This is used since some features (in particular ClassLabel) have some logic during encoding.
1283
+
1284
+ To avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be encoded.
1285
+ If the first element needs to be encoded, then all the elements of the list will be encoded, otherwise they'll stay the same.
1286
+ """
1287
+ # Nested structures: we allow dict, list/tuples, sequences
1288
+ if isinstance(schema, dict):
1289
+ if level == 0 and obj is None:
1290
+ raise ValueError("Got None but expected a dictionary instead")
1291
+ return (
1292
+ {k: encode_nested_example(schema[k], obj.get(k), level=level + 1) for k in schema}
1293
+ if obj is not None
1294
+ else None
1295
+ )
1296
+
1297
+ elif isinstance(schema, (list, tuple)):
1298
+ sub_schema = schema[0]
1299
+ if obj is None:
1300
+ return None
1301
+ elif isinstance(obj, np.ndarray):
1302
+ return encode_nested_example(schema, obj.tolist())
1303
+ else:
1304
+ if len(obj) > 0:
1305
+ for first_elmt in obj:
1306
+ if _check_non_null_non_empty_recursive(first_elmt, sub_schema):
1307
+ break
1308
+ if encode_nested_example(sub_schema, first_elmt, level=level + 1) != first_elmt:
1309
+ return [encode_nested_example(sub_schema, o, level=level + 1) for o in obj]
1310
+ return list(obj)
1311
+ elif isinstance(schema, LargeList):
1312
+ if obj is None:
1313
+ return None
1314
+ else:
1315
+ if len(obj) > 0:
1316
+ sub_schema = schema.feature
1317
+ for first_elmt in obj:
1318
+ if _check_non_null_non_empty_recursive(first_elmt, sub_schema):
1319
+ break
1320
+ if encode_nested_example(sub_schema, first_elmt, level=level + 1) != first_elmt:
1321
+ return [encode_nested_example(sub_schema, o, level=level + 1) for o in obj]
1322
+ return list(obj)
1323
+ elif isinstance(schema, Sequence):
1324
+ if obj is None:
1325
+ return None
1326
+ # We allow to reverse list of dict => dict of list for compatibility with tfds
1327
+ if isinstance(schema.feature, dict):
1328
+ # dict of list to fill
1329
+ list_dict = {}
1330
+ if isinstance(obj, (list, tuple)):
1331
+ # obj is a list of dict
1332
+ for k in schema.feature:
1333
+ list_dict[k] = [encode_nested_example(schema.feature[k], o.get(k), level=level + 1) for o in obj]
1334
+ return list_dict
1335
+ else:
1336
+ # obj is a single dict
1337
+ for k in schema.feature:
1338
+ list_dict[k] = (
1339
+ [encode_nested_example(schema.feature[k], o, level=level + 1) for o in obj[k]]
1340
+ if k in obj
1341
+ else None
1342
+ )
1343
+ return list_dict
1344
+ # schema.feature is not a dict
1345
+ if isinstance(obj, str): # don't interpret a string as a list
1346
+ raise ValueError(f"Got a string but expected a list instead: '{obj}'")
1347
+ else:
1348
+ if len(obj) > 0:
1349
+ for first_elmt in obj:
1350
+ if _check_non_null_non_empty_recursive(first_elmt, schema.feature):
1351
+ break
1352
+ # be careful when comparing tensors here
1353
+ if (
1354
+ not (isinstance(first_elmt, list) or np.isscalar(first_elmt))
1355
+ or encode_nested_example(schema.feature, first_elmt, level=level + 1) != first_elmt
1356
+ ):
1357
+ return [encode_nested_example(schema.feature, o, level=level + 1) for o in obj]
1358
+ return list(obj)
1359
+ # Object with special encoding:
1360
+ # ClassLabel will convert from string to int, TranslationVariableLanguages does some checks
1361
+ elif hasattr(schema, "encode_example"):
1362
+ return schema.encode_example(obj) if obj is not None else None
1363
+ # Other object should be directly convertible to a native Arrow type (like Translation and Translation)
1364
+ return obj
1365
+
1366
+
1367
+ def decode_nested_example(schema, obj, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None):
1368
+ """Decode a nested example.
1369
+ This is used since some features (in particular Audio and Image) have some logic during decoding.
1370
+
1371
+ To avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be decoded.
1372
+ If the first element needs to be decoded, then all the elements of the list will be decoded, otherwise they'll stay the same.
1373
+ """
1374
+ # Nested structures: we allow dict, list/tuples, sequences
1375
+ if isinstance(schema, dict):
1376
+ return (
1377
+ {k: decode_nested_example(sub_schema, sub_obj) for k, (sub_schema, sub_obj) in zip_dict(schema, obj)}
1378
+ if obj is not None
1379
+ else None
1380
+ )
1381
+ elif isinstance(schema, (list, tuple)):
1382
+ sub_schema = schema[0]
1383
+ if obj is None:
1384
+ return None
1385
+ else:
1386
+ if len(obj) > 0:
1387
+ for first_elmt in obj:
1388
+ if _check_non_null_non_empty_recursive(first_elmt, sub_schema):
1389
+ break
1390
+ if decode_nested_example(sub_schema, first_elmt) != first_elmt:
1391
+ return [decode_nested_example(sub_schema, o) for o in obj]
1392
+ return list(obj)
1393
+ elif isinstance(schema, LargeList):
1394
+ if obj is None:
1395
+ return None
1396
+ else:
1397
+ sub_schema = schema.feature
1398
+ if len(obj) > 0:
1399
+ for first_elmt in obj:
1400
+ if _check_non_null_non_empty_recursive(first_elmt, sub_schema):
1401
+ break
1402
+ if decode_nested_example(sub_schema, first_elmt) != first_elmt:
1403
+ return [decode_nested_example(sub_schema, o) for o in obj]
1404
+ return list(obj)
1405
+ elif isinstance(schema, Sequence):
1406
+ # We allow to reverse list of dict => dict of list for compatibility with tfds
1407
+ if isinstance(schema.feature, dict):
1408
+ return {k: decode_nested_example([schema.feature[k]], obj[k]) for k in schema.feature}
1409
+ else:
1410
+ return decode_nested_example([schema.feature], obj)
1411
+ # Object with special decoding:
1412
+ elif hasattr(schema, "decode_example") and getattr(schema, "decode", True):
1413
+ # we pass the token to read and decode files from private repositories in streaming mode
1414
+ return schema.decode_example(obj, token_per_repo_id=token_per_repo_id) if obj is not None else None
1415
+ return obj
1416
+
1417
+
1418
+ _FEATURE_TYPES: dict[str, FeatureType] = {
1419
+ Value.__name__: Value,
1420
+ ClassLabel.__name__: ClassLabel,
1421
+ Translation.__name__: Translation,
1422
+ TranslationVariableLanguages.__name__: TranslationVariableLanguages,
1423
+ LargeList.__name__: LargeList,
1424
+ Sequence.__name__: Sequence,
1425
+ Array2D.__name__: Array2D,
1426
+ Array3D.__name__: Array3D,
1427
+ Array4D.__name__: Array4D,
1428
+ Array5D.__name__: Array5D,
1429
+ Audio.__name__: Audio,
1430
+ Image.__name__: Image,
1431
+ Video.__name__: Video,
1432
+ Pdf.__name__: Pdf,
1433
+ }
1434
+
1435
+
1436
+ @experimental
1437
+ def register_feature(
1438
+ feature_cls: type,
1439
+ feature_type: str,
1440
+ ):
1441
+ """
1442
+ Register a Feature object using a name and class.
1443
+ This function must be used on a Feature class.
1444
+ """
1445
+ if feature_type in _FEATURE_TYPES:
1446
+ logger.warning(
1447
+ f"Overwriting feature type '{feature_type}' ({_FEATURE_TYPES[feature_type].__name__} -> {feature_cls.__name__})"
1448
+ )
1449
+ _FEATURE_TYPES[feature_type] = feature_cls
1450
+
1451
+
1452
+ def generate_from_dict(obj: Any):
1453
+ """Regenerate the nested feature object from a deserialized dict.
1454
+ We use the '_type' fields to get the dataclass name to load.
1455
+
1456
+ generate_from_dict is the recursive helper for Features.from_dict, and allows for a convenient constructor syntax
1457
+ to define features from deserialized JSON dictionaries. This function is used in particular when deserializing
1458
+ a :class:`DatasetInfo` that was dumped to a JSON object. This acts as an analogue to
1459
+ :meth:`Features.from_arrow_schema` and handles the recursive field-by-field instantiation, but doesn't require any
1460
+ mapping to/from pyarrow, except for the fact that it takes advantage of the mapping of pyarrow primitive dtypes
1461
+ that :class:`Value` automatically performs.
1462
+ """
1463
+ # Nested structures: we allow dict, list/tuples, sequences
1464
+ if isinstance(obj, list):
1465
+ return [generate_from_dict(value) for value in obj]
1466
+ # Otherwise we have a dict or a dataclass
1467
+ if "_type" not in obj or isinstance(obj["_type"], dict):
1468
+ return {key: generate_from_dict(value) for key, value in obj.items()}
1469
+ obj = dict(obj)
1470
+ _type = obj.pop("_type")
1471
+ class_type = _FEATURE_TYPES.get(_type, None) or globals().get(_type, None)
1472
+
1473
+ if class_type is None:
1474
+ raise ValueError(f"Feature type '{_type}' not found. Available feature types: {list(_FEATURE_TYPES.keys())}")
1475
+
1476
+ if class_type == LargeList:
1477
+ feature = obj.pop("feature")
1478
+ return LargeList(feature=generate_from_dict(feature), **obj)
1479
+ if class_type == Sequence:
1480
+ feature = obj.pop("feature")
1481
+ return Sequence(feature=generate_from_dict(feature), **obj)
1482
+
1483
+ field_names = {f.name for f in fields(class_type)}
1484
+ return class_type(**{k: v for k, v in obj.items() if k in field_names})
1485
+
1486
+
1487
+ def generate_from_arrow_type(pa_type: pa.DataType) -> FeatureType:
1488
+ """
1489
+ generate_from_arrow_type accepts an arrow DataType and returns a datasets FeatureType to be used as the type for
1490
+ a single field.
1491
+
1492
+ This is the high-level arrow->datasets type conversion and is inverted by get_nested_type().
1493
+
1494
+ This operates at the individual *field* level, whereas Features.from_arrow_schema() operates at the
1495
+ full schema level and holds the methods that represent the bijection from Features<->pyarrow.Schema
1496
+ """
1497
+ if isinstance(pa_type, pa.StructType):
1498
+ return {field.name: generate_from_arrow_type(field.type) for field in pa_type}
1499
+ elif isinstance(pa_type, pa.FixedSizeListType):
1500
+ return Sequence(feature=generate_from_arrow_type(pa_type.value_type), length=pa_type.list_size)
1501
+ elif isinstance(pa_type, pa.ListType):
1502
+ feature = generate_from_arrow_type(pa_type.value_type)
1503
+ if isinstance(feature, (dict, tuple, list)):
1504
+ return [feature]
1505
+ return Sequence(feature=feature)
1506
+ elif isinstance(pa_type, pa.LargeListType):
1507
+ feature = generate_from_arrow_type(pa_type.value_type)
1508
+ return LargeList(feature=feature)
1509
+ elif isinstance(pa_type, _ArrayXDExtensionType):
1510
+ array_feature = [None, None, Array2D, Array3D, Array4D, Array5D][pa_type.ndims]
1511
+ return array_feature(shape=pa_type.shape, dtype=pa_type.value_type)
1512
+ elif isinstance(pa_type, pa.DataType):
1513
+ return Value(dtype=_arrow_to_datasets_dtype(pa_type))
1514
+ else:
1515
+ raise ValueError(f"Cannot convert {pa_type} to a Feature type.")
1516
+
1517
+
1518
+ def numpy_to_pyarrow_listarray(arr: np.ndarray, type: pa.DataType = None) -> pa.ListArray:
1519
+ """Build a PyArrow ListArray from a multidimensional NumPy array"""
1520
+ arr = np.array(arr)
1521
+ values = pa.array(arr.flatten(), type=type)
1522
+ for i in range(arr.ndim - 1):
1523
+ n_offsets = reduce(mul, arr.shape[: arr.ndim - i - 1], 1)
1524
+ step_offsets = arr.shape[arr.ndim - i - 1]
1525
+ offsets = pa.array(np.arange(n_offsets + 1) * step_offsets, type=pa.int32())
1526
+ values = pa.ListArray.from_arrays(offsets, values)
1527
+ return values
1528
+
1529
+
1530
+ def list_of_pa_arrays_to_pyarrow_listarray(l_arr: list[Optional[pa.Array]]) -> pa.ListArray:
1531
+ null_mask = np.array([arr is None for arr in l_arr])
1532
+ null_indices = np.arange(len(null_mask))[null_mask] - np.arange(np.sum(null_mask))
1533
+ l_arr = [arr for arr in l_arr if arr is not None]
1534
+ offsets = np.cumsum(
1535
+ [0] + [len(arr) for arr in l_arr], dtype=object
1536
+ ) # convert to dtype object to allow None insertion
1537
+ offsets = np.insert(offsets, null_indices, None)
1538
+ offsets = pa.array(offsets, type=pa.int32())
1539
+ values = pa.concat_arrays(l_arr)
1540
+ return pa.ListArray.from_arrays(offsets, values)
1541
+
1542
+
1543
+ def list_of_np_array_to_pyarrow_listarray(l_arr: list[np.ndarray], type: pa.DataType = None) -> pa.ListArray:
1544
+ """Build a PyArrow ListArray from a possibly nested list of NumPy arrays"""
1545
+ if len(l_arr) > 0:
1546
+ return list_of_pa_arrays_to_pyarrow_listarray(
1547
+ [numpy_to_pyarrow_listarray(arr, type=type) if arr is not None else None for arr in l_arr]
1548
+ )
1549
+ else:
1550
+ return pa.array([], type=type)
1551
+
1552
+
1553
+ def contains_any_np_array(data: Any):
1554
+ """Return `True` if data is a NumPy ndarray or (recursively) if first non-null value in list is a NumPy ndarray.
1555
+
1556
+ Args:
1557
+ data (Any): Data.
1558
+
1559
+ Returns:
1560
+ bool
1561
+ """
1562
+ if isinstance(data, np.ndarray):
1563
+ return True
1564
+ elif isinstance(data, list):
1565
+ return contains_any_np_array(first_non_null_value(data)[1])
1566
+ else:
1567
+ return False
1568
+
1569
+
1570
+ def any_np_array_to_pyarrow_listarray(data: Union[np.ndarray, list], type: pa.DataType = None) -> pa.ListArray:
1571
+ """Convert to PyArrow ListArray either a NumPy ndarray or (recursively) a list that may contain any NumPy ndarray.
1572
+
1573
+ Args:
1574
+ data (Union[np.ndarray, List]): Data.
1575
+ type (pa.DataType): Explicit PyArrow DataType passed to coerce the ListArray data type.
1576
+
1577
+ Returns:
1578
+ pa.ListArray
1579
+ """
1580
+ if isinstance(data, np.ndarray):
1581
+ return numpy_to_pyarrow_listarray(data, type=type)
1582
+ elif isinstance(data, list):
1583
+ return list_of_pa_arrays_to_pyarrow_listarray([any_np_array_to_pyarrow_listarray(i, type=type) for i in data])
1584
+
1585
+
1586
+ def to_pyarrow_listarray(data: Any, pa_type: _ArrayXDExtensionType) -> pa.Array:
1587
+ """Convert to PyArrow ListArray.
1588
+
1589
+ Args:
1590
+ data (Any): Sequence, iterable, np.ndarray or pd.Series.
1591
+ pa_type (_ArrayXDExtensionType): Any of the ArrayNDExtensionType.
1592
+
1593
+ Returns:
1594
+ pyarrow.Array
1595
+ """
1596
+ if contains_any_np_array(data):
1597
+ return any_np_array_to_pyarrow_listarray(data, type=pa_type.value_type)
1598
+ else:
1599
+ return pa.array(data, pa_type.storage_dtype)
1600
+
1601
+
1602
+ def _visit(feature: FeatureType, func: Callable[[FeatureType], Optional[FeatureType]]) -> FeatureType:
1603
+ """Visit a (possibly nested) feature.
1604
+
1605
+ Args:
1606
+ feature (FeatureType): the feature type to be checked
1607
+ Returns:
1608
+ visited feature (FeatureType)
1609
+ """
1610
+ if isinstance(feature, Features):
1611
+ out = func(Features({k: _visit(f, func) for k, f in feature.items()}))
1612
+ elif isinstance(feature, dict):
1613
+ out = func({k: _visit(f, func) for k, f in feature.items()})
1614
+ elif isinstance(feature, (list, tuple)):
1615
+ out = func([_visit(feature[0], func)])
1616
+ elif isinstance(feature, LargeList):
1617
+ out = func(LargeList(_visit(feature.feature, func)))
1618
+ elif isinstance(feature, Sequence):
1619
+ out = func(Sequence(_visit(feature.feature, func), length=feature.length))
1620
+ else:
1621
+ out = func(feature)
1622
+ return feature if out is None else out
1623
+
1624
+
1625
+ _VisitPath = list[Union[str, Literal[0]]]
1626
+
1627
+
1628
+ def _visit_with_path(
1629
+ feature: FeatureType, func: Callable[[FeatureType, _VisitPath], Optional[FeatureType]], visit_path: _VisitPath = []
1630
+ ) -> FeatureType:
1631
+ """Visit a (possibly nested) feature with its path in the Feature object.
1632
+
1633
+ A path in a nested feature object is the list of keys that need to be
1634
+ sequentially accessed to get to the sub-feature.
1635
+
1636
+ For example:
1637
+ - ["foo"] corresponds to the column "foo"
1638
+ - ["foo", 0] corresponds to the sub-feature of the lists in "foo"
1639
+ - ["foo", "bar"] corresponds to the sub-feature of the dicts in "foo" with key "bar"
1640
+
1641
+ Args:
1642
+ feature (`FeatureType`): the feature type to be checked.
1643
+
1644
+ Returns:
1645
+ `FeatureType`: the visited feature.
1646
+ """
1647
+ if isinstance(feature, Sequence) and isinstance(feature.feature, dict):
1648
+ feature = {k: [f] for k, f in feature.feature.items()}
1649
+ # ^ Sequence of dicts is special, it must be converted to a dict of lists (see https://huggingface.co/docs/datasets/v2.16.1/en/package_reference/main_classes#datasets.Features)
1650
+ if isinstance(feature, Features):
1651
+ out = func(Features({k: _visit_with_path(f, func, visit_path + [k]) for k, f in feature.items()}), visit_path)
1652
+ elif isinstance(feature, dict):
1653
+ out = func({k: _visit_with_path(f, func, visit_path + [k]) for k, f in feature.items()}, visit_path)
1654
+ elif isinstance(feature, (list, tuple)):
1655
+ out = func([_visit_with_path(feature[0], func, visit_path + [0])], visit_path)
1656
+ elif isinstance(feature, Sequence):
1657
+ out = func(
1658
+ Sequence(_visit_with_path(feature.feature, func, visit_path + [0]), length=feature.length), visit_path
1659
+ )
1660
+ elif isinstance(feature, LargeList):
1661
+ out = func(LargeList(_visit_with_path(feature.feature, func, visit_path + [0])), visit_path)
1662
+ else:
1663
+ out = func(feature, visit_path)
1664
+ return feature if out is None else out
1665
+
1666
+
1667
+ def require_decoding(feature: FeatureType, ignore_decode_attribute: bool = False) -> bool:
1668
+ """Check if a (possibly nested) feature requires decoding.
1669
+
1670
+ Args:
1671
+ feature (FeatureType): the feature type to be checked
1672
+ ignore_decode_attribute (:obj:`bool`, default ``False``): Whether to ignore the current value
1673
+ of the `decode` attribute of the decodable feature types.
1674
+ Returns:
1675
+ :obj:`bool`
1676
+ """
1677
+ if isinstance(feature, dict):
1678
+ return any(require_decoding(f) for f in feature.values())
1679
+ elif isinstance(feature, (list, tuple)):
1680
+ return require_decoding(feature[0])
1681
+ elif isinstance(feature, LargeList):
1682
+ return require_decoding(feature.feature)
1683
+ elif isinstance(feature, Sequence):
1684
+ return require_decoding(feature.feature)
1685
+ else:
1686
+ return hasattr(feature, "decode_example") and (
1687
+ getattr(feature, "decode", True) if not ignore_decode_attribute else True
1688
+ )
1689
+
1690
+
1691
+ def require_storage_cast(feature: FeatureType) -> bool:
1692
+ """Check if a (possibly nested) feature requires storage casting.
1693
+
1694
+ Args:
1695
+ feature (FeatureType): the feature type to be checked
1696
+ Returns:
1697
+ :obj:`bool`
1698
+ """
1699
+ if isinstance(feature, dict):
1700
+ return any(require_storage_cast(f) for f in feature.values())
1701
+ elif isinstance(feature, (list, tuple)):
1702
+ return require_storage_cast(feature[0])
1703
+ elif isinstance(feature, LargeList):
1704
+ return require_storage_cast(feature.feature)
1705
+ elif isinstance(feature, Sequence):
1706
+ return require_storage_cast(feature.feature)
1707
+ else:
1708
+ return hasattr(feature, "cast_storage")
1709
+
1710
+
1711
+ def require_storage_embed(feature: FeatureType) -> bool:
1712
+ """Check if a (possibly nested) feature requires embedding data into storage.
1713
+
1714
+ Args:
1715
+ feature (FeatureType): the feature type to be checked
1716
+ Returns:
1717
+ :obj:`bool`
1718
+ """
1719
+ if isinstance(feature, dict):
1720
+ return any(require_storage_cast(f) for f in feature.values())
1721
+ elif isinstance(feature, (list, tuple)):
1722
+ return require_storage_cast(feature[0])
1723
+ elif isinstance(feature, LargeList):
1724
+ return require_storage_cast(feature.feature)
1725
+ elif isinstance(feature, Sequence):
1726
+ return require_storage_cast(feature.feature)
1727
+ else:
1728
+ return hasattr(feature, "embed_storage")
1729
+
1730
+
1731
+ def keep_features_dicts_synced(func):
1732
+ """
1733
+ Wrapper to keep the secondary dictionary, which tracks whether keys are decodable, of the :class:`datasets.Features` object
1734
+ in sync with the main dictionary.
1735
+ """
1736
+
1737
+ @wraps(func)
1738
+ def wrapper(*args, **kwargs):
1739
+ if args:
1740
+ self: "Features" = args[0]
1741
+ args = args[1:]
1742
+ else:
1743
+ self: "Features" = kwargs.pop("self")
1744
+ out = func(self, *args, **kwargs)
1745
+ assert hasattr(self, "_column_requires_decoding")
1746
+ self._column_requires_decoding = {col: require_decoding(feature) for col, feature in self.items()}
1747
+ return out
1748
+
1749
+ wrapper._decorator_name_ = "_keep_dicts_synced"
1750
+ return wrapper
1751
+
1752
+
1753
+ class Features(dict):
1754
+ """A special dictionary that defines the internal structure of a dataset.
1755
+
1756
+ Instantiated with a dictionary of type `dict[str, FieldType]`, where keys are the desired column names,
1757
+ and values are the type of that column.
1758
+
1759
+ `FieldType` can be one of the following:
1760
+ - [`Value`] feature specifies a single data type value, e.g. `int64` or `string`.
1761
+ - [`ClassLabel`] feature specifies a predefined set of classes which can have labels associated to them and
1762
+ will be stored as integers in the dataset.
1763
+ - Python `dict` specifies a composite feature containing a mapping of sub-fields to sub-features.
1764
+ It's possible to have nested fields of nested fields in an arbitrary manner.
1765
+ - Python `list`, [`LargeList`] or [`Sequence`] specifies a composite feature containing a sequence of
1766
+ sub-features, all of the same feature type.
1767
+
1768
+ <Tip>
1769
+
1770
+ A [`Sequence`] with an internal dictionary feature will be automatically converted into a dictionary of
1771
+ lists. This behavior is implemented to have a compatibility layer with the TensorFlow Datasets library but may be
1772
+ un-wanted in some cases. If you don't want this behavior, you can use a Python `list` or a [`LargeList`]
1773
+ instead of the [`Sequence`].
1774
+
1775
+ </Tip>
1776
+
1777
+ - [`Array2D`], [`Array3D`], [`Array4D`] or [`Array5D`] feature for multidimensional arrays.
1778
+ - [`Audio`] feature to store the absolute path to an audio file or a dictionary with the relative path
1779
+ to an audio file ("path" key) and its bytes content ("bytes" key). This feature extracts the audio data.
1780
+ - [`Image`] feature to store the absolute path to an image file, an `np.ndarray` object, a `PIL.Image.Image` object
1781
+ or a dictionary with the relative path to an image file ("path" key) and its bytes content ("bytes" key).
1782
+ This feature extracts the image data.
1783
+ - [`Translation`] or [`TranslationVariableLanguages`] feature specific to Machine Translation.
1784
+ """
1785
+
1786
+ def __init__(*args, **kwargs):
1787
+ # self not in the signature to allow passing self as a kwarg
1788
+ if not args:
1789
+ raise TypeError("descriptor '__init__' of 'Features' object needs an argument")
1790
+ self, *args = args
1791
+ super(Features, self).__init__(*args, **kwargs)
1792
+ self._column_requires_decoding: dict[str, bool] = {
1793
+ col: require_decoding(feature) for col, feature in self.items()
1794
+ }
1795
+
1796
+ __setitem__ = keep_features_dicts_synced(dict.__setitem__)
1797
+ __delitem__ = keep_features_dicts_synced(dict.__delitem__)
1798
+ update = keep_features_dicts_synced(dict.update)
1799
+ setdefault = keep_features_dicts_synced(dict.setdefault)
1800
+ pop = keep_features_dicts_synced(dict.pop)
1801
+ popitem = keep_features_dicts_synced(dict.popitem)
1802
+ clear = keep_features_dicts_synced(dict.clear)
1803
+
1804
+ def __reduce__(self):
1805
+ return Features, (dict(self),)
1806
+
1807
+ @property
1808
+ def type(self):
1809
+ """
1810
+ Features field types.
1811
+
1812
+ Returns:
1813
+ :obj:`pyarrow.DataType`
1814
+ """
1815
+ return get_nested_type(self)
1816
+
1817
+ @property
1818
+ def arrow_schema(self):
1819
+ """
1820
+ Features schema.
1821
+
1822
+ Returns:
1823
+ :obj:`pyarrow.Schema`
1824
+ """
1825
+ hf_metadata = {"info": {"features": self.to_dict()}}
1826
+ return pa.schema(self.type).with_metadata({"huggingface": json.dumps(hf_metadata)})
1827
+
1828
+ @classmethod
1829
+ def from_arrow_schema(cls, pa_schema: pa.Schema) -> "Features":
1830
+ """
1831
+ Construct [`Features`] from Arrow Schema.
1832
+ It also checks the schema metadata for Hugging Face Datasets features.
1833
+ Non-nullable fields are not supported and set to nullable.
1834
+
1835
+ Also, pa.dictionary is not supported and it uses its underlying type instead.
1836
+ Therefore datasets convert DictionaryArray objects to their actual values.
1837
+
1838
+ Args:
1839
+ pa_schema (`pyarrow.Schema`):
1840
+ Arrow Schema.
1841
+
1842
+ Returns:
1843
+ [`Features`]
1844
+ """
1845
+ # try to load features from the arrow schema metadata
1846
+ metadata_features = Features()
1847
+ if pa_schema.metadata is not None and b"huggingface" in pa_schema.metadata:
1848
+ metadata = json.loads(pa_schema.metadata[b"huggingface"].decode())
1849
+ if "info" in metadata and "features" in metadata["info"] and metadata["info"]["features"] is not None:
1850
+ metadata_features = Features.from_dict(metadata["info"]["features"])
1851
+ metadata_features_schema = metadata_features.arrow_schema
1852
+ obj = {
1853
+ field.name: (
1854
+ metadata_features[field.name]
1855
+ if field.name in metadata_features and metadata_features_schema.field(field.name) == field
1856
+ else generate_from_arrow_type(field.type)
1857
+ )
1858
+ for field in pa_schema
1859
+ }
1860
+ return cls(**obj)
1861
+
1862
+ @classmethod
1863
+ def from_dict(cls, dic) -> "Features":
1864
+ """
1865
+ Construct [`Features`] from dict.
1866
+
1867
+ Regenerate the nested feature object from a deserialized dict.
1868
+ We use the `_type` key to infer the dataclass name of the feature `FieldType`.
1869
+
1870
+ It allows for a convenient constructor syntax
1871
+ to define features from deserialized JSON dictionaries. This function is used in particular when deserializing
1872
+ a [`DatasetInfo`] that was dumped to a JSON object. This acts as an analogue to
1873
+ [`Features.from_arrow_schema`] and handles the recursive field-by-field instantiation, but doesn't require
1874
+ any mapping to/from pyarrow, except for the fact that it takes advantage of the mapping of pyarrow primitive
1875
+ dtypes that [`Value`] automatically performs.
1876
+
1877
+ Args:
1878
+ dic (`dict[str, Any]`):
1879
+ Python dictionary.
1880
+
1881
+ Returns:
1882
+ `Features`
1883
+
1884
+ Example::
1885
+ >>> Features.from_dict({'_type': {'dtype': 'string', 'id': None, '_type': 'Value'}})
1886
+ {'_type': Value(dtype='string', id=None)}
1887
+ """
1888
+ obj = generate_from_dict(dic)
1889
+ return cls(**obj)
1890
+
1891
+ def to_dict(self):
1892
+ return asdict(self)
1893
+
1894
+ def _to_yaml_list(self) -> list:
1895
+ # we compute the YAML list from the dict representation that is used for JSON dump
1896
+ yaml_data = self.to_dict()
1897
+
1898
+ def simplify(feature: dict) -> dict:
1899
+ if not isinstance(feature, dict):
1900
+ raise TypeError(f"Expected a dict but got a {type(feature)}: {feature}")
1901
+
1902
+ for list_type in ["large_list", "list", "sequence"]:
1903
+ #
1904
+ # list_type: -> list_type: int32
1905
+ # dtype: int32 ->
1906
+ #
1907
+ if isinstance(feature.get(list_type), dict) and list(feature[list_type]) == ["dtype"]:
1908
+ feature[list_type] = feature[list_type]["dtype"]
1909
+
1910
+ #
1911
+ # list_type: -> list_type:
1912
+ # struct: -> - name: foo
1913
+ # - name: foo -> dtype: int32
1914
+ # dtype: int32 ->
1915
+ #
1916
+ if isinstance(feature.get(list_type), dict) and list(feature[list_type]) == ["struct"]:
1917
+ feature[list_type] = feature[list_type]["struct"]
1918
+
1919
+ #
1920
+ # class_label: -> class_label:
1921
+ # names: -> names:
1922
+ # - negative -> '0': negative
1923
+ # - positive -> '1': positive
1924
+ #
1925
+ if isinstance(feature.get("class_label"), dict) and isinstance(feature["class_label"].get("names"), list):
1926
+ # server-side requirement: keys must be strings
1927
+ feature["class_label"]["names"] = {
1928
+ str(label_id): label_name for label_id, label_name in enumerate(feature["class_label"]["names"])
1929
+ }
1930
+ return feature
1931
+
1932
+ def to_yaml_inner(obj: Union[dict, list]) -> dict:
1933
+ if isinstance(obj, dict):
1934
+ _type = obj.pop("_type", None)
1935
+ if _type == "LargeList":
1936
+ _feature = obj.pop("feature")
1937
+ return simplify({"large_list": to_yaml_inner(_feature), **obj})
1938
+ elif _type == "Sequence":
1939
+ _feature = obj.pop("feature")
1940
+ return simplify({"sequence": to_yaml_inner(_feature), **obj})
1941
+ elif _type == "Value":
1942
+ return obj
1943
+ elif _type and not obj:
1944
+ return {"dtype": camelcase_to_snakecase(_type)}
1945
+ elif _type:
1946
+ return {"dtype": simplify({camelcase_to_snakecase(_type): obj})}
1947
+ else:
1948
+ return {"struct": [{"name": name, **to_yaml_inner(_feature)} for name, _feature in obj.items()]}
1949
+ elif isinstance(obj, list):
1950
+ return simplify({"list": simplify(to_yaml_inner(obj[0]))})
1951
+ elif isinstance(obj, tuple):
1952
+ return to_yaml_inner(list(obj))
1953
+ else:
1954
+ raise TypeError(f"Expected a dict or a list but got {type(obj)}: {obj}")
1955
+
1956
+ def to_yaml_types(obj: dict) -> dict:
1957
+ if isinstance(obj, dict):
1958
+ return {k: to_yaml_types(v) for k, v in obj.items()}
1959
+ elif isinstance(obj, list):
1960
+ return [to_yaml_types(v) for v in obj]
1961
+ elif isinstance(obj, tuple):
1962
+ return to_yaml_types(list(obj))
1963
+ else:
1964
+ return obj
1965
+
1966
+ return to_yaml_types(to_yaml_inner(yaml_data)["struct"])
1967
+
1968
+ @classmethod
1969
+ def _from_yaml_list(cls, yaml_data: list) -> "Features":
1970
+ yaml_data = copy.deepcopy(yaml_data)
1971
+
1972
+ # we convert the list obtained from YAML data into the dict representation that is used for JSON dump
1973
+
1974
+ def unsimplify(feature: dict) -> dict:
1975
+ if not isinstance(feature, dict):
1976
+ raise TypeError(f"Expected a dict but got a {type(feature)}: {feature}")
1977
+
1978
+ for list_type in ["large_list", "list", "sequence"]:
1979
+ #
1980
+ # list_type: int32 -> list_type:
1981
+ # -> dtype: int32
1982
+ #
1983
+ if isinstance(feature.get(list_type), str):
1984
+ feature[list_type] = {"dtype": feature[list_type]}
1985
+
1986
+ #
1987
+ # class_label: -> class_label:
1988
+ # names: -> names:
1989
+ # '0': negative -> - negative
1990
+ # '1': positive -> - positive
1991
+ #
1992
+ if isinstance(feature.get("class_label"), dict) and isinstance(feature["class_label"].get("names"), dict):
1993
+ label_ids = sorted(feature["class_label"]["names"], key=int)
1994
+ if label_ids and [int(label_id) for label_id in label_ids] != list(range(int(label_ids[-1]) + 1)):
1995
+ raise ValueError(
1996
+ f"ClassLabel expected a value for all label ids [0:{int(label_ids[-1]) + 1}] but some ids are missing."
1997
+ )
1998
+ feature["class_label"]["names"] = [feature["class_label"]["names"][label_id] for label_id in label_ids]
1999
+ return feature
2000
+
2001
+ def from_yaml_inner(obj: Union[dict, list]) -> Union[dict, list]:
2002
+ if isinstance(obj, dict):
2003
+ if not obj:
2004
+ return {}
2005
+ _type = next(iter(obj))
2006
+ if _type == "large_list":
2007
+ _feature = unsimplify(obj).pop(_type)
2008
+ return {"feature": from_yaml_inner(_feature), **obj, "_type": "LargeList"}
2009
+ if _type == "sequence":
2010
+ _feature = unsimplify(obj).pop(_type)
2011
+ return {"feature": from_yaml_inner(_feature), **obj, "_type": "Sequence"}
2012
+ if _type == "list":
2013
+ return [from_yaml_inner(unsimplify(obj)[_type])]
2014
+ if _type == "struct":
2015
+ return from_yaml_inner(obj["struct"])
2016
+ elif _type == "dtype":
2017
+ if isinstance(obj["dtype"], str):
2018
+ # e.g. int32, float64, string, audio, image
2019
+ try:
2020
+ Value(obj["dtype"])
2021
+ return {**obj, "_type": "Value"}
2022
+ except ValueError:
2023
+ # e.g. Audio, Image, ArrayXD
2024
+ return {"_type": snakecase_to_camelcase(obj["dtype"])}
2025
+ else:
2026
+ return from_yaml_inner(obj["dtype"])
2027
+ else:
2028
+ return {"_type": snakecase_to_camelcase(_type), **unsimplify(obj)[_type]}
2029
+ elif isinstance(obj, list):
2030
+ names = [_feature.pop("name") for _feature in obj]
2031
+ return {name: from_yaml_inner(_feature) for name, _feature in zip(names, obj)}
2032
+ else:
2033
+ raise TypeError(f"Expected a dict or a list but got {type(obj)}: {obj}")
2034
+
2035
+ return cls.from_dict(from_yaml_inner(yaml_data))
2036
+
2037
+ def encode_example(self, example):
2038
+ """
2039
+ Encode example into a format for Arrow.
2040
+
2041
+ Args:
2042
+ example (`dict[str, Any]`):
2043
+ Data in a Dataset row.
2044
+
2045
+ Returns:
2046
+ `dict[str, Any]`
2047
+ """
2048
+ example = cast_to_python_objects(example)
2049
+ return encode_nested_example(self, example)
2050
+
2051
+ def encode_column(self, column, column_name: str):
2052
+ """
2053
+ Encode column into a format for Arrow.
2054
+
2055
+ Args:
2056
+ column (`list[Any]`):
2057
+ Data in a Dataset column.
2058
+ column_name (`str`):
2059
+ Dataset column name.
2060
+
2061
+ Returns:
2062
+ `list[Any]`
2063
+ """
2064
+ column = cast_to_python_objects(column)
2065
+ return [encode_nested_example(self[column_name], obj, level=1) for obj in column]
2066
+
2067
+ def encode_batch(self, batch):
2068
+ """
2069
+ Encode batch into a format for Arrow.
2070
+
2071
+ Args:
2072
+ batch (`dict[str, list[Any]]`):
2073
+ Data in a Dataset batch.
2074
+
2075
+ Returns:
2076
+ `dict[str, list[Any]]`
2077
+ """
2078
+ encoded_batch = {}
2079
+ if set(batch) != set(self):
2080
+ raise ValueError(f"Column mismatch between batch {set(batch)} and features {set(self)}")
2081
+ for key, column in batch.items():
2082
+ column = cast_to_python_objects(column)
2083
+ encoded_batch[key] = [encode_nested_example(self[key], obj, level=1) for obj in column]
2084
+ return encoded_batch
2085
+
2086
+ def decode_example(self, example: dict, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None):
2087
+ """Decode example with custom feature decoding.
2088
+
2089
+ Args:
2090
+ example (`dict[str, Any]`):
2091
+ Dataset row data.
2092
+ token_per_repo_id (`dict`, *optional*):
2093
+ To access and decode audio or image files from private repositories on the Hub, you can pass
2094
+ a dictionary `repo_id (str) -> token (bool or str)`.
2095
+
2096
+ Returns:
2097
+ `dict[str, Any]`
2098
+ """
2099
+
2100
+ return {
2101
+ column_name: decode_nested_example(feature, value, token_per_repo_id=token_per_repo_id)
2102
+ if self._column_requires_decoding[column_name]
2103
+ else value
2104
+ for column_name, (feature, value) in zip_dict(
2105
+ {key: value for key, value in self.items() if key in example}, example
2106
+ )
2107
+ }
2108
+
2109
+ def decode_column(self, column: list, column_name: str):
2110
+ """Decode column with custom feature decoding.
2111
+
2112
+ Args:
2113
+ column (`list[Any]`):
2114
+ Dataset column data.
2115
+ column_name (`str`):
2116
+ Dataset column name.
2117
+
2118
+ Returns:
2119
+ `list[Any]`
2120
+ """
2121
+ return (
2122
+ [decode_nested_example(self[column_name], value) if value is not None else None for value in column]
2123
+ if self._column_requires_decoding[column_name]
2124
+ else column
2125
+ )
2126
+
2127
+ def decode_batch(self, batch: dict, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None):
2128
+ """Decode batch with custom feature decoding.
2129
+
2130
+ Args:
2131
+ batch (`dict[str, list[Any]]`):
2132
+ Dataset batch data.
2133
+ token_per_repo_id (`dict`, *optional*):
2134
+ To access and decode audio or image files from private repositories on the Hub, you can pass
2135
+ a dictionary repo_id (str) -> token (bool or str)
2136
+
2137
+ Returns:
2138
+ `dict[str, list[Any]]`
2139
+ """
2140
+ decoded_batch = {}
2141
+ for column_name, column in batch.items():
2142
+ decoded_batch[column_name] = (
2143
+ [
2144
+ decode_nested_example(self[column_name], value, token_per_repo_id=token_per_repo_id)
2145
+ if value is not None
2146
+ else None
2147
+ for value in column
2148
+ ]
2149
+ if self._column_requires_decoding[column_name]
2150
+ else column
2151
+ )
2152
+ return decoded_batch
2153
+
2154
+ def copy(self) -> "Features":
2155
+ """
2156
+ Make a deep copy of [`Features`].
2157
+
2158
+ Returns:
2159
+ [`Features`]
2160
+
2161
+ Example:
2162
+
2163
+ ```py
2164
+ >>> from datasets import load_dataset
2165
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
2166
+ >>> copy_of_features = ds.features.copy()
2167
+ >>> copy_of_features
2168
+ {'label': ClassLabel(names=['neg', 'pos'], id=None),
2169
+ 'text': Value(dtype='string', id=None)}
2170
+ ```
2171
+ """
2172
+ return copy.deepcopy(self)
2173
+
2174
+ def reorder_fields_as(self, other: "Features") -> "Features":
2175
+ """
2176
+ Reorder Features fields to match the field order of other [`Features`].
2177
+
2178
+ The order of the fields is important since it matters for the underlying arrow data.
2179
+ Re-ordering the fields allows to make the underlying arrow data type match.
2180
+
2181
+ Args:
2182
+ other ([`Features`]):
2183
+ The other [`Features`] to align with.
2184
+
2185
+ Returns:
2186
+ [`Features`]
2187
+
2188
+ Example::
2189
+
2190
+ >>> from datasets import Features, Sequence, Value
2191
+ >>> # let's say we have two features with a different order of nested fields (for a and b for example)
2192
+ >>> f1 = Features({"root": Sequence({"a": Value("string"), "b": Value("string")})})
2193
+ >>> f2 = Features({"root": {"b": Sequence(Value("string")), "a": Sequence(Value("string"))}})
2194
+ >>> assert f1.type != f2.type
2195
+ >>> # re-ordering keeps the base structure (here Sequence is defined at the root level), but makes the fields order match
2196
+ >>> f1.reorder_fields_as(f2)
2197
+ {'root': Sequence(feature={'b': Value(dtype='string', id=None), 'a': Value(dtype='string', id=None)}, length=-1, id=None)}
2198
+ >>> assert f1.reorder_fields_as(f2).type == f2.type
2199
+ """
2200
+
2201
+ def recursive_reorder(source, target, stack=""):
2202
+ stack_position = " at " + stack[1:] if stack else ""
2203
+ if isinstance(target, Sequence):
2204
+ target = target.feature
2205
+ if isinstance(target, dict):
2206
+ target = {k: [v] for k, v in target.items()}
2207
+ else:
2208
+ target = [target]
2209
+ if isinstance(source, Sequence):
2210
+ sequence_kwargs = vars(source).copy()
2211
+ source = sequence_kwargs.pop("feature")
2212
+ if isinstance(source, dict):
2213
+ source = {k: [v] for k, v in source.items()}
2214
+ reordered = recursive_reorder(source, target, stack)
2215
+ return Sequence({k: v[0] for k, v in reordered.items()}, **sequence_kwargs)
2216
+ else:
2217
+ source = [source]
2218
+ reordered = recursive_reorder(source, target, stack)
2219
+ return Sequence(reordered[0], **sequence_kwargs)
2220
+ elif isinstance(source, dict):
2221
+ if not isinstance(target, dict):
2222
+ raise ValueError(f"Type mismatch: between {source} and {target}" + stack_position)
2223
+ if sorted(source) != sorted(target):
2224
+ message = (
2225
+ f"Keys mismatch: between {source} (source) and {target} (target).\n"
2226
+ f"{source.keys() - target.keys()} are missing from target "
2227
+ f"and {target.keys() - source.keys()} are missing from source" + stack_position
2228
+ )
2229
+ raise ValueError(message)
2230
+ return {key: recursive_reorder(source[key], target[key], stack + f".{key}") for key in target}
2231
+ elif isinstance(source, list):
2232
+ if not isinstance(target, list):
2233
+ raise ValueError(f"Type mismatch: between {source} and {target}" + stack_position)
2234
+ if len(source) != len(target):
2235
+ raise ValueError(f"Length mismatch: between {source} and {target}" + stack_position)
2236
+ return [recursive_reorder(source[i], target[i], stack + ".<list>") for i in range(len(target))]
2237
+ elif isinstance(source, LargeList):
2238
+ if not isinstance(target, LargeList):
2239
+ raise ValueError(f"Type mismatch: between {source} and {target}" + stack_position)
2240
+ return LargeList(recursive_reorder(source.feature, target.feature, stack))
2241
+ else:
2242
+ return source
2243
+
2244
+ return Features(recursive_reorder(self, other))
2245
+
2246
+ def flatten(self, max_depth=16) -> "Features":
2247
+ """Flatten the features. Every dictionary column is removed and is replaced by
2248
+ all the subfields it contains. The new fields are named by concatenating the
2249
+ name of the original column and the subfield name like this: `<original>.<subfield>`.
2250
+
2251
+ If a column contains nested dictionaries, then all the lower-level subfields names are
2252
+ also concatenated to form new columns: `<original>.<subfield>.<subsubfield>`, etc.
2253
+
2254
+ Returns:
2255
+ [`Features`]:
2256
+ The flattened features.
2257
+
2258
+ Example:
2259
+
2260
+ ```py
2261
+ >>> from datasets import load_dataset
2262
+ >>> ds = load_dataset("rajpurkar/squad", split="train")
2263
+ >>> ds.features.flatten()
2264
+ {'answers.answer_start': Sequence(feature=Value(dtype='int32', id=None), length=-1, id=None),
2265
+ 'answers.text': Sequence(feature=Value(dtype='string', id=None), length=-1, id=None),
2266
+ 'context': Value(dtype='string', id=None),
2267
+ 'id': Value(dtype='string', id=None),
2268
+ 'question': Value(dtype='string', id=None),
2269
+ 'title': Value(dtype='string', id=None)}
2270
+ ```
2271
+ """
2272
+ for depth in range(1, max_depth):
2273
+ no_change = True
2274
+ flattened = self.copy()
2275
+ for column_name, subfeature in self.items():
2276
+ if isinstance(subfeature, dict):
2277
+ no_change = False
2278
+ flattened.update({f"{column_name}.{k}": v for k, v in subfeature.items()})
2279
+ del flattened[column_name]
2280
+ elif isinstance(subfeature, Sequence) and isinstance(subfeature.feature, dict):
2281
+ no_change = False
2282
+ flattened.update(
2283
+ {
2284
+ f"{column_name}.{k}": Sequence(v) if not isinstance(v, dict) else [v]
2285
+ for k, v in subfeature.feature.items()
2286
+ }
2287
+ )
2288
+ del flattened[column_name]
2289
+ elif hasattr(subfeature, "flatten") and subfeature.flatten() != subfeature:
2290
+ no_change = False
2291
+ flattened.update({f"{column_name}.{k}": v for k, v in subfeature.flatten().items()})
2292
+ del flattened[column_name]
2293
+ self = flattened
2294
+ if no_change:
2295
+ break
2296
+ return self
2297
+
2298
+
2299
+ def _align_features(features_list: list[Features]) -> list[Features]:
2300
+ """Align dictionaries of features so that the keys that are found in multiple dictionaries share the same feature."""
2301
+ name2feature = {}
2302
+ for features in features_list:
2303
+ for k, v in features.items():
2304
+ if k in name2feature and isinstance(v, dict):
2305
+ # Recursively align features.
2306
+ name2feature[k] = _align_features([name2feature[k], v])[0]
2307
+ elif k not in name2feature or (isinstance(name2feature[k], Value) and name2feature[k].dtype == "null"):
2308
+ name2feature[k] = v
2309
+
2310
+ return [Features({k: name2feature[k] for k in features.keys()}) for features in features_list]
2311
+
2312
+
2313
+ def _check_if_features_can_be_aligned(features_list: list[Features]):
2314
+ """Check if the dictionaries of features can be aligned.
2315
+
2316
+ Two dictonaries of features can be aligned if the keys they share have the same type or some of them is of type `Value("null")`.
2317
+ """
2318
+ name2feature = {}
2319
+ for features in features_list:
2320
+ for k, v in features.items():
2321
+ if k not in name2feature or (isinstance(name2feature[k], Value) and name2feature[k].dtype == "null"):
2322
+ name2feature[k] = v
2323
+
2324
+ for features in features_list:
2325
+ for k, v in features.items():
2326
+ if isinstance(v, dict) and isinstance(name2feature[k], dict):
2327
+ # Deep checks for structure.
2328
+ _check_if_features_can_be_aligned([name2feature[k], v])
2329
+ elif not (isinstance(v, Value) and v.dtype == "null") and name2feature[k] != v:
2330
+ raise ValueError(
2331
+ f'The features can\'t be aligned because the key {k} of features {features} has unexpected type - {v} (expected either {name2feature[k]} or Value("null").'
2332
+ )
lib/python3.12/site-packages/datasets/features/image.py ADDED
@@ -0,0 +1,382 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import sys
3
+ import warnings
4
+ from dataclasses import dataclass, field
5
+ from io import BytesIO
6
+ from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union
7
+
8
+ import numpy as np
9
+ import pyarrow as pa
10
+
11
+ from .. import config
12
+ from ..download.download_config import DownloadConfig
13
+ from ..table import array_cast
14
+ from ..utils.file_utils import is_local_path, xopen
15
+ from ..utils.py_utils import first_non_null_value, no_op_if_value_is_null, string_to_dict
16
+
17
+
18
+ if TYPE_CHECKING:
19
+ import PIL.Image
20
+
21
+ from .features import FeatureType
22
+
23
+
24
+ _IMAGE_COMPRESSION_FORMATS: Optional[list[str]] = None
25
+ _NATIVE_BYTEORDER = "<" if sys.byteorder == "little" else ">"
26
+ # Origin: https://github.com/python-pillow/Pillow/blob/698951e19e19972aeed56df686868f1329981c12/src/PIL/Image.py#L3126 minus "|i1" which values are not preserved correctly when saving and loading an image
27
+ _VALID_IMAGE_ARRAY_DTPYES = [
28
+ np.dtype("|b1"),
29
+ np.dtype("|u1"),
30
+ np.dtype("<u2"),
31
+ np.dtype(">u2"),
32
+ np.dtype("<i2"),
33
+ np.dtype(">i2"),
34
+ np.dtype("<u4"),
35
+ np.dtype(">u4"),
36
+ np.dtype("<i4"),
37
+ np.dtype(">i4"),
38
+ np.dtype("<f4"),
39
+ np.dtype(">f4"),
40
+ np.dtype("<f8"),
41
+ np.dtype(">f8"),
42
+ ]
43
+
44
+
45
+ @dataclass
46
+ class Image:
47
+ """Image [`Feature`] to read image data from an image file.
48
+
49
+ Input: The Image feature accepts as input:
50
+ - A `str`: Absolute path to the image file (i.e. random access is allowed).
51
+ - A `dict` with the keys:
52
+
53
+ - `path`: String with relative path of the image file to the archive file.
54
+ - `bytes`: Bytes of the image file.
55
+
56
+ This is useful for archived files with sequential access.
57
+
58
+ - An `np.ndarray`: NumPy array representing an image.
59
+ - A `PIL.Image.Image`: PIL image object.
60
+
61
+ Args:
62
+ mode (`str`, *optional*):
63
+ The mode to convert the image to. If `None`, the native mode of the image is used.
64
+ decode (`bool`, defaults to `True`):
65
+ Whether to decode the image data. If `False`,
66
+ returns the underlying dictionary in the format `{"path": image_path, "bytes": image_bytes}`.
67
+
68
+ Examples:
69
+
70
+ ```py
71
+ >>> from datasets import load_dataset, Image
72
+ >>> ds = load_dataset("AI-Lab-Makerere/beans", split="train")
73
+ >>> ds.features["image"]
74
+ Image(decode=True, id=None)
75
+ >>> ds[0]["image"]
76
+ <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=500x500 at 0x15E52E7F0>
77
+ >>> ds = ds.cast_column('image', Image(decode=False))
78
+ {'bytes': None,
79
+ 'path': '/root/.cache/huggingface/datasets/downloads/extracted/b0a21163f78769a2cf11f58dfc767fb458fc7cea5c05dccc0144a2c0f0bc1292/train/healthy/healthy_train.85.jpg'}
80
+ ```
81
+ """
82
+
83
+ mode: Optional[str] = None
84
+ decode: bool = True
85
+ id: Optional[str] = None
86
+ # Automatically constructed
87
+ dtype: ClassVar[str] = "PIL.Image.Image"
88
+ pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()})
89
+ _type: str = field(default="Image", init=False, repr=False)
90
+
91
+ def __call__(self):
92
+ return self.pa_type
93
+
94
+ def encode_example(self, value: Union[str, bytes, bytearray, dict, np.ndarray, "PIL.Image.Image"]) -> dict:
95
+ """Encode example into a format for Arrow.
96
+
97
+ Args:
98
+ value (`str`, `np.ndarray`, `PIL.Image.Image` or `dict`):
99
+ Data passed as input to Image feature.
100
+
101
+ Returns:
102
+ `dict` with "path" and "bytes" fields
103
+ """
104
+ if config.PIL_AVAILABLE:
105
+ import PIL.Image
106
+ else:
107
+ raise ImportError("To support encoding images, please install 'Pillow'.")
108
+
109
+ if isinstance(value, list):
110
+ value = np.array(value)
111
+
112
+ if isinstance(value, str):
113
+ return {"path": value, "bytes": None}
114
+ elif isinstance(value, (bytes, bytearray)):
115
+ return {"path": None, "bytes": value}
116
+ elif isinstance(value, np.ndarray):
117
+ # convert the image array to PNG/TIFF bytes
118
+ return encode_np_array(value)
119
+ elif isinstance(value, PIL.Image.Image):
120
+ # convert the PIL image to bytes (default format is PNG/TIFF)
121
+ return encode_pil_image(value)
122
+ elif value.get("path") is not None and os.path.isfile(value["path"]):
123
+ # we set "bytes": None to not duplicate the data if they're already available locally
124
+ return {"bytes": None, "path": value.get("path")}
125
+ elif value.get("bytes") is not None or value.get("path") is not None:
126
+ # store the image bytes, and path is used to infer the image format using the file extension
127
+ return {"bytes": value.get("bytes"), "path": value.get("path")}
128
+ else:
129
+ raise ValueError(
130
+ f"An image sample should have one of 'path' or 'bytes' but they are missing or None in {value}."
131
+ )
132
+
133
+ def decode_example(self, value: dict, token_per_repo_id=None) -> "PIL.Image.Image":
134
+ """Decode example image file into image data.
135
+
136
+ Args:
137
+ value (`str` or `dict`):
138
+ A string with the absolute image file path, a dictionary with
139
+ keys:
140
+
141
+ - `path`: String with absolute or relative image file path.
142
+ - `bytes`: The bytes of the image file.
143
+ token_per_repo_id (`dict`, *optional*):
144
+ To access and decode
145
+ image files from private repositories on the Hub, you can pass
146
+ a dictionary repo_id (`str`) -> token (`bool` or `str`).
147
+
148
+ Returns:
149
+ `PIL.Image.Image`
150
+ """
151
+ if not self.decode:
152
+ raise RuntimeError("Decoding is disabled for this feature. Please use Image(decode=True) instead.")
153
+
154
+ if config.PIL_AVAILABLE:
155
+ import PIL.Image
156
+ import PIL.ImageOps
157
+ else:
158
+ raise ImportError("To support decoding images, please install 'Pillow'.")
159
+
160
+ if token_per_repo_id is None:
161
+ token_per_repo_id = {}
162
+
163
+ path, bytes_ = value["path"], value["bytes"]
164
+ if bytes_ is None:
165
+ if path is None:
166
+ raise ValueError(f"An image should have one of 'path' or 'bytes' but both are None in {value}.")
167
+ else:
168
+ if is_local_path(path):
169
+ image = PIL.Image.open(path)
170
+ else:
171
+ source_url = path.split("::")[-1]
172
+ pattern = (
173
+ config.HUB_DATASETS_URL
174
+ if source_url.startswith(config.HF_ENDPOINT)
175
+ else config.HUB_DATASETS_HFFS_URL
176
+ )
177
+ source_url_fields = string_to_dict(source_url, pattern)
178
+ token = (
179
+ token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None
180
+ )
181
+ download_config = DownloadConfig(token=token)
182
+ with xopen(path, "rb", download_config=download_config) as f:
183
+ bytes_ = BytesIO(f.read())
184
+ image = PIL.Image.open(bytes_)
185
+ else:
186
+ image = PIL.Image.open(BytesIO(bytes_))
187
+ image.load() # to avoid "Too many open files" errors
188
+ if image.getexif().get(PIL.Image.ExifTags.Base.Orientation) is not None:
189
+ image = PIL.ImageOps.exif_transpose(image)
190
+ if self.mode and self.mode != image.mode:
191
+ image = image.convert(self.mode)
192
+ return image
193
+
194
+ def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]:
195
+ """If in the decodable state, return the feature itself, otherwise flatten the feature into a dictionary."""
196
+ from .features import Value
197
+
198
+ return (
199
+ self
200
+ if self.decode
201
+ else {
202
+ "bytes": Value("binary"),
203
+ "path": Value("string"),
204
+ }
205
+ )
206
+
207
+ def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray, pa.ListArray]) -> pa.StructArray:
208
+ """Cast an Arrow array to the Image arrow storage type.
209
+ The Arrow types that can be converted to the Image pyarrow storage type are:
210
+
211
+ - `pa.string()` - it must contain the "path" data
212
+ - `pa.binary()` - it must contain the image bytes
213
+ - `pa.struct({"bytes": pa.binary()})`
214
+ - `pa.struct({"path": pa.string()})`
215
+ - `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter
216
+ - `pa.list(*)` - it must contain the image array data
217
+
218
+ Args:
219
+ storage (`Union[pa.StringArray, pa.StructArray, pa.ListArray]`):
220
+ PyArrow array to cast.
221
+
222
+ Returns:
223
+ `pa.StructArray`: Array in the Image arrow storage type, that is
224
+ `pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
225
+ """
226
+ if pa.types.is_string(storage.type):
227
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
228
+ storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null())
229
+ elif pa.types.is_binary(storage.type):
230
+ path_array = pa.array([None] * len(storage), type=pa.string())
231
+ storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null())
232
+ elif pa.types.is_struct(storage.type):
233
+ if storage.type.get_field_index("bytes") >= 0:
234
+ bytes_array = storage.field("bytes")
235
+ else:
236
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
237
+ if storage.type.get_field_index("path") >= 0:
238
+ path_array = storage.field("path")
239
+ else:
240
+ path_array = pa.array([None] * len(storage), type=pa.string())
241
+ storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null())
242
+ elif pa.types.is_list(storage.type):
243
+ bytes_array = pa.array(
244
+ [encode_np_array(np.array(arr))["bytes"] if arr is not None else None for arr in storage.to_pylist()],
245
+ type=pa.binary(),
246
+ )
247
+ path_array = pa.array([None] * len(storage), type=pa.string())
248
+ storage = pa.StructArray.from_arrays(
249
+ [bytes_array, path_array], ["bytes", "path"], mask=bytes_array.is_null()
250
+ )
251
+ return array_cast(storage, self.pa_type)
252
+
253
+ def embed_storage(self, storage: pa.StructArray) -> pa.StructArray:
254
+ """Embed image files into the Arrow array.
255
+
256
+ Args:
257
+ storage (`pa.StructArray`):
258
+ PyArrow array to embed.
259
+
260
+ Returns:
261
+ `pa.StructArray`: Array in the Image arrow storage type, that is
262
+ `pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
263
+ """
264
+
265
+ @no_op_if_value_is_null
266
+ def path_to_bytes(path):
267
+ with xopen(path, "rb") as f:
268
+ bytes_ = f.read()
269
+ return bytes_
270
+
271
+ bytes_array = pa.array(
272
+ [
273
+ (path_to_bytes(x["path"]) if x["bytes"] is None else x["bytes"]) if x is not None else None
274
+ for x in storage.to_pylist()
275
+ ],
276
+ type=pa.binary(),
277
+ )
278
+ path_array = pa.array(
279
+ [os.path.basename(path) if path is not None else None for path in storage.field("path").to_pylist()],
280
+ type=pa.string(),
281
+ )
282
+ storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=bytes_array.is_null())
283
+ return array_cast(storage, self.pa_type)
284
+
285
+
286
+ def list_image_compression_formats() -> list[str]:
287
+ if config.PIL_AVAILABLE:
288
+ import PIL.Image
289
+ else:
290
+ raise ImportError("To support encoding images, please install 'Pillow'.")
291
+
292
+ global _IMAGE_COMPRESSION_FORMATS
293
+ if _IMAGE_COMPRESSION_FORMATS is None:
294
+ PIL.Image.init()
295
+ _IMAGE_COMPRESSION_FORMATS = list(set(PIL.Image.OPEN.keys()) & set(PIL.Image.SAVE.keys()))
296
+ return _IMAGE_COMPRESSION_FORMATS
297
+
298
+
299
+ def image_to_bytes(image: "PIL.Image.Image") -> bytes:
300
+ """Convert a PIL Image object to bytes using native compression if possible, otherwise use PNG/TIFF compression."""
301
+ buffer = BytesIO()
302
+ if image.format in list_image_compression_formats():
303
+ format = image.format
304
+ else:
305
+ format = "PNG" if image.mode in ["1", "L", "LA", "RGB", "RGBA"] else "TIFF"
306
+ image.save(buffer, format=format)
307
+ return buffer.getvalue()
308
+
309
+
310
+ def encode_pil_image(image: "PIL.Image.Image") -> dict:
311
+ if hasattr(image, "filename") and image.filename != "":
312
+ return {"path": image.filename, "bytes": None}
313
+ else:
314
+ return {"path": None, "bytes": image_to_bytes(image)}
315
+
316
+
317
+ def encode_np_array(array: np.ndarray) -> dict:
318
+ if config.PIL_AVAILABLE:
319
+ import PIL.Image
320
+ else:
321
+ raise ImportError("To support encoding images, please install 'Pillow'.")
322
+
323
+ dtype = array.dtype
324
+ dtype_byteorder = dtype.byteorder if dtype.byteorder != "=" else _NATIVE_BYTEORDER
325
+ dtype_kind = dtype.kind
326
+ dtype_itemsize = dtype.itemsize
327
+
328
+ dest_dtype = None
329
+
330
+ # Multi-channel array case (only np.dtype("|u1") is allowed)
331
+ if array.shape[2:]:
332
+ if dtype_kind not in ["u", "i"]:
333
+ raise TypeError(
334
+ f"Unsupported array dtype {dtype} for image encoding. Only {dest_dtype} is supported for multi-channel arrays."
335
+ )
336
+ dest_dtype = np.dtype("|u1")
337
+ if dtype != dest_dtype:
338
+ warnings.warn(f"Downcasting array dtype {dtype} to {dest_dtype} to be compatible with 'Pillow'")
339
+ # Exact match
340
+ elif dtype in _VALID_IMAGE_ARRAY_DTPYES:
341
+ dest_dtype = dtype
342
+ else: # Downcast the type within the kind (np.can_cast(from_type, to_type, casting="same_kind") doesn't behave as expected, so do it manually)
343
+ while dtype_itemsize >= 1:
344
+ dtype_str = dtype_byteorder + dtype_kind + str(dtype_itemsize)
345
+ if np.dtype(dtype_str) in _VALID_IMAGE_ARRAY_DTPYES:
346
+ dest_dtype = np.dtype(dtype_str)
347
+ warnings.warn(f"Downcasting array dtype {dtype} to {dest_dtype} to be compatible with 'Pillow'")
348
+ break
349
+ else:
350
+ dtype_itemsize //= 2
351
+ if dest_dtype is None:
352
+ raise TypeError(
353
+ f"Cannot downcast dtype {dtype} to a valid image dtype. Valid image dtypes: {_VALID_IMAGE_ARRAY_DTPYES}"
354
+ )
355
+
356
+ image = PIL.Image.fromarray(array.astype(dest_dtype))
357
+ return {"path": None, "bytes": image_to_bytes(image)}
358
+
359
+
360
+ def objects_to_list_of_image_dicts(
361
+ objs: Union[list[str], list[dict], list[np.ndarray], list["PIL.Image.Image"]],
362
+ ) -> list[dict]:
363
+ """Encode a list of objects into a format suitable for creating an extension array of type `ImageExtensionType`."""
364
+ if config.PIL_AVAILABLE:
365
+ import PIL.Image
366
+ else:
367
+ raise ImportError("To support encoding images, please install 'Pillow'.")
368
+
369
+ if objs:
370
+ _, obj = first_non_null_value(objs)
371
+ if isinstance(obj, str):
372
+ return [{"path": obj, "bytes": None} if obj is not None else None for obj in objs]
373
+ if isinstance(obj, np.ndarray):
374
+ obj_to_image_dict_func = no_op_if_value_is_null(encode_np_array)
375
+ return [obj_to_image_dict_func(obj) for obj in objs]
376
+ elif isinstance(obj, PIL.Image.Image):
377
+ obj_to_image_dict_func = no_op_if_value_is_null(encode_pil_image)
378
+ return [obj_to_image_dict_func(obj) for obj in objs]
379
+ else:
380
+ return objs
381
+ else:
382
+ return objs
lib/python3.12/site-packages/datasets/features/pdf.py ADDED
@@ -0,0 +1,238 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from dataclasses import dataclass, field
3
+ from io import BytesIO
4
+ from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union
5
+
6
+ import pyarrow as pa
7
+
8
+ from .. import config
9
+ from ..download.download_config import DownloadConfig
10
+ from ..table import array_cast
11
+ from ..utils.file_utils import is_local_path, xopen
12
+ from ..utils.py_utils import string_to_dict
13
+
14
+
15
+ if TYPE_CHECKING:
16
+ import pdfplumber
17
+
18
+ from .features import FeatureType
19
+
20
+
21
+ def pdf_to_bytes(pdf: "pdfplumber.pdf.PDF") -> bytes:
22
+ """Convert a pdfplumber.pdf.PDF object to bytes."""
23
+ with BytesIO() as buffer:
24
+ for page in pdf.pages:
25
+ buffer.write(page.pdf.stream)
26
+ return buffer.getvalue()
27
+
28
+
29
+ @dataclass
30
+ class Pdf:
31
+ """
32
+ **Experimental.**
33
+ Pdf [`Feature`] to read pdf documents from a pdf file.
34
+
35
+ Input: The Pdf feature accepts as input:
36
+ - A `str`: Absolute path to the pdf file (i.e. random access is allowed).
37
+ - A `dict` with the keys:
38
+ - `path`: String with relative path of the pdf file in a dataset repository.
39
+ - `bytes`: Bytes of the pdf file.
40
+ This is useful for archived files with sequential access.
41
+
42
+ - A `pdfplumber.pdf.PDF`: pdfplumber pdf object.
43
+
44
+ Args:
45
+ mode (`str`, *optional*):
46
+ The mode to convert the pdf to. If `None`, the native mode of the pdf is used.
47
+ decode (`bool`, defaults to `True`):
48
+ Whether to decode the pdf data. If `False`,
49
+ returns the underlying dictionary in the format `{"path": pdf_path, "bytes": pdf_bytes}`.
50
+
51
+ Examples:
52
+
53
+ ```py
54
+ >>> from datasets import Dataset, Pdf
55
+ >>> ds = Dataset.from_dict({"pdf": ["path/to/pdf/file.pdf"]}).cast_column("pdf", Pdf())
56
+ >>> ds.features["pdf"]
57
+ Pdf(decode=True, id=None)
58
+ >>> ds[0]["pdf"]
59
+ <pdfplumber.pdf.PDF object at 0x7f8a1c2d8f40>
60
+ >>> ds = ds.cast_column("pdf", Pdf(decode=False))
61
+ >>> ds[0]["pdf"]
62
+ {'bytes': None,
63
+ 'path': 'path/to/pdf/file.pdf'}
64
+ ```
65
+ """
66
+
67
+ decode: bool = True
68
+ id: Optional[str] = None
69
+
70
+ # Automatically constructed
71
+ dtype: ClassVar[str] = "pdfplumber.pdf.PDF"
72
+ pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()})
73
+ _type: str = field(default="Pdf", init=False, repr=False)
74
+
75
+ def __call__(self):
76
+ return self.pa_type
77
+
78
+ def encode_example(self, value: Union[str, bytes, bytearray, dict, "pdfplumber.pdf.PDF"]) -> dict:
79
+ """Encode example into a format for Arrow.
80
+
81
+ Args:
82
+ value (`str`, `bytes`, `pdfplumber.pdf.PDF` or `dict`):
83
+ Data passed as input to Pdf feature.
84
+
85
+ Returns:
86
+ `dict` with "path" and "bytes" fields
87
+ """
88
+ if config.PDFPLUMBER_AVAILABLE:
89
+ import pdfplumber
90
+ else:
91
+ pdfplumber = None
92
+
93
+ if isinstance(value, str):
94
+ return {"path": value, "bytes": None}
95
+ elif isinstance(value, (bytes, bytearray)):
96
+ return {"path": None, "bytes": value}
97
+ elif pdfplumber is not None and isinstance(value, pdfplumber.pdf.PDF):
98
+ # convert the pdfplumber.pdf.PDF to bytes
99
+ return encode_pdfplumber_pdf(value)
100
+ elif value.get("path") is not None and os.path.isfile(value["path"]):
101
+ # we set "bytes": None to not duplicate the data if they're already available locally
102
+ return {"bytes": None, "path": value.get("path")}
103
+ elif value.get("bytes") is not None or value.get("path") is not None:
104
+ # store the pdf bytes, and path is used to infer the pdf format using the file extension
105
+ return {"bytes": value.get("bytes"), "path": value.get("path")}
106
+ else:
107
+ raise ValueError(
108
+ f"A pdf sample should have one of 'path' or 'bytes' but they are missing or None in {value}."
109
+ )
110
+
111
+ def decode_example(self, value: dict, token_per_repo_id=None) -> "pdfplumber.pdf.PDF":
112
+ """Decode example pdf file into pdf data.
113
+
114
+ Args:
115
+ value (`str` or `dict`):
116
+ A string with the absolute pdf file path, a dictionary with
117
+ keys:
118
+
119
+ - `path`: String with absolute or relative pdf file path.
120
+ - `bytes`: The bytes of the pdf file.
121
+
122
+ token_per_repo_id (`dict`, *optional*):
123
+ To access and decode pdf files from private repositories on
124
+ the Hub, you can pass a dictionary
125
+ repo_id (`str`) -> token (`bool` or `str`).
126
+
127
+ Returns:
128
+ `pdfplumber.pdf.PDF`
129
+ """
130
+ if not self.decode:
131
+ raise RuntimeError("Decoding is disabled for this feature. Please use Pdf(decode=True) instead.")
132
+
133
+ if config.PDFPLUMBER_AVAILABLE:
134
+ import pdfplumber
135
+ else:
136
+ raise ImportError("To support decoding pdfs, please install 'pdfplumber'.")
137
+
138
+ if token_per_repo_id is None:
139
+ token_per_repo_id = {}
140
+
141
+ path, bytes_ = value["path"], value["bytes"]
142
+ if bytes_ is None:
143
+ if path is None:
144
+ raise ValueError(f"A pdf should have one of 'path' or 'bytes' but both are None in {value}.")
145
+ else:
146
+ if is_local_path(path):
147
+ pdf = pdfplumber.open(path)
148
+ else:
149
+ source_url = path.split("::")[-1]
150
+ pattern = (
151
+ config.HUB_DATASETS_URL
152
+ if source_url.startswith(config.HF_ENDPOINT)
153
+ else config.HUB_DATASETS_HFFS_URL
154
+ )
155
+ try:
156
+ repo_id = string_to_dict(source_url, pattern)["repo_id"]
157
+ token = token_per_repo_id.get(repo_id)
158
+ except ValueError:
159
+ token = None
160
+ download_config = DownloadConfig(token=token)
161
+ f = xopen(path, "rb", download_config=download_config)
162
+ return pdfplumber.open(f)
163
+ else:
164
+ with pdfplumber.open(BytesIO(bytes_)) as p:
165
+ pdf = p
166
+
167
+ return pdf
168
+
169
+ def flatten(self) -> Union["FeatureType", Dict[str, "FeatureType"]]:
170
+ """If in the decodable state, return the feature itself, otherwise flatten the feature into a dictionary."""
171
+ from .features import Value
172
+
173
+ return (
174
+ self
175
+ if self.decode
176
+ else {
177
+ "bytes": Value("binary"),
178
+ "path": Value("string"),
179
+ }
180
+ )
181
+
182
+ def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray, pa.ListArray]) -> pa.StructArray:
183
+ """Cast an Arrow array to the Pdf arrow storage type.
184
+ The Arrow types that can be converted to the Pdf pyarrow storage type are:
185
+
186
+ - `pa.string()` - it must contain the "path" data
187
+ - `pa.binary()` - it must contain the image bytes
188
+ - `pa.struct({"bytes": pa.binary()})`
189
+ - `pa.struct({"path": pa.string()})`
190
+ - `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter
191
+ - `pa.list(*)` - it must contain the pdf array data
192
+
193
+ Args:
194
+ storage (`Union[pa.StringArray, pa.StructArray, pa.ListArray]`):
195
+ PyArrow array to cast.
196
+
197
+ Returns:
198
+ `pa.StructArray`: Array in the Pdf arrow storage type, that is
199
+ `pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
200
+ """
201
+ if pa.types.is_string(storage.type):
202
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
203
+ storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null())
204
+ elif pa.types.is_binary(storage.type):
205
+ path_array = pa.array([None] * len(storage), type=pa.string())
206
+ storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null())
207
+ elif pa.types.is_struct(storage.type):
208
+ if storage.type.get_field_index("bytes") >= 0:
209
+ bytes_array = storage.field("bytes")
210
+ else:
211
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
212
+ if storage.type.get_field_index("path") >= 0:
213
+ path_array = storage.field("path")
214
+ else:
215
+ path_array = pa.array([None] * len(storage), type=pa.string())
216
+ storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null())
217
+ return array_cast(storage, self.pa_type)
218
+
219
+
220
+ def encode_pdfplumber_pdf(pdf: "pdfplumber.pdf.PDF") -> dict:
221
+ """
222
+ Encode a pdfplumber.pdf.PDF object into a dictionary.
223
+
224
+ If the PDF has an associated file path, returns the path. Otherwise, serializes
225
+ the PDF content into bytes.
226
+
227
+ Args:
228
+ pdf (pdfplumber.pdf.PDF): A pdfplumber PDF object.
229
+
230
+ Returns:
231
+ dict: A dictionary with "path" or "bytes" field.
232
+ """
233
+ if hasattr(pdf, "stream") and hasattr(pdf.stream, "name") and pdf.stream.name:
234
+ # Return the path if the PDF has an associated file path
235
+ return {"path": pdf.stream.name, "bytes": None}
236
+ else:
237
+ # Convert the PDF to bytes if no path is available
238
+ return {"path": None, "bytes": pdf_to_bytes(pdf)}
lib/python3.12/site-packages/datasets/features/translation.py ADDED
@@ -0,0 +1,129 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass, field
2
+ from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union
3
+
4
+ import pyarrow as pa
5
+
6
+
7
+ if TYPE_CHECKING:
8
+ from .features import FeatureType
9
+
10
+
11
+ @dataclass
12
+ class Translation:
13
+ """`Feature` for translations with fixed languages per example.
14
+ Here for compatibility with tfds.
15
+
16
+ Args:
17
+ languages (`dict`):
18
+ A dictionary for each example mapping string language codes to string translations.
19
+
20
+ Example:
21
+
22
+ ```python
23
+ >>> # At construction time:
24
+ >>> datasets.features.Translation(languages=['en', 'fr', 'de'])
25
+ >>> # During data generation:
26
+ >>> yield {
27
+ ... 'en': 'the cat',
28
+ ... 'fr': 'le chat',
29
+ ... 'de': 'die katze'
30
+ ... }
31
+ ```
32
+ """
33
+
34
+ languages: list[str]
35
+ id: Optional[str] = None
36
+ # Automatically constructed
37
+ dtype: ClassVar[str] = "dict"
38
+ pa_type: ClassVar[Any] = None
39
+ _type: str = field(default="Translation", init=False, repr=False)
40
+
41
+ def __call__(self):
42
+ return pa.struct({lang: pa.string() for lang in sorted(self.languages)})
43
+
44
+ def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]:
45
+ """Flatten the Translation feature into a dictionary."""
46
+ from .features import Value
47
+
48
+ return {k: Value("string") for k in sorted(self.languages)}
49
+
50
+
51
+ @dataclass
52
+ class TranslationVariableLanguages:
53
+ """`Feature` for translations with variable languages per example.
54
+ Here for compatibility with tfds.
55
+
56
+ Args:
57
+ languages (`dict`):
58
+ A dictionary for each example mapping string language codes to one or more string translations.
59
+ The languages present may vary from example to example.
60
+
61
+ Returns:
62
+ - `language` or `translation` (variable-length 1D `tf.Tensor` of `tf.string`):
63
+ Language codes sorted in ascending order or plain text translations, sorted to align with language codes.
64
+
65
+ Example:
66
+
67
+ ```python
68
+ >>> # At construction time:
69
+ >>> datasets.features.TranslationVariableLanguages(languages=['en', 'fr', 'de'])
70
+ >>> # During data generation:
71
+ >>> yield {
72
+ ... 'en': 'the cat',
73
+ ... 'fr': ['le chat', 'la chatte,']
74
+ ... 'de': 'die katze'
75
+ ... }
76
+ >>> # Tensor returned :
77
+ >>> {
78
+ ... 'language': ['en', 'de', 'fr', 'fr'],
79
+ ... 'translation': ['the cat', 'die katze', 'la chatte', 'le chat'],
80
+ ... }
81
+ ```
82
+ """
83
+
84
+ languages: Optional[list] = None
85
+ num_languages: Optional[int] = None
86
+ id: Optional[str] = None
87
+ # Automatically constructed
88
+ dtype: ClassVar[str] = "dict"
89
+ pa_type: ClassVar[Any] = None
90
+ _type: str = field(default="TranslationVariableLanguages", init=False, repr=False)
91
+
92
+ def __post_init__(self):
93
+ self.languages = sorted(set(self.languages)) if self.languages else None
94
+ self.num_languages = len(self.languages) if self.languages else None
95
+
96
+ def __call__(self):
97
+ return pa.struct({"language": pa.list_(pa.string()), "translation": pa.list_(pa.string())})
98
+
99
+ def encode_example(self, translation_dict):
100
+ lang_set = set(self.languages)
101
+ if set(translation_dict) == {"language", "translation"}:
102
+ return translation_dict
103
+ elif self.languages and set(translation_dict) - lang_set:
104
+ raise ValueError(
105
+ f"Some languages in example ({', '.join(sorted(set(translation_dict) - lang_set))}) are not in valid set ({', '.join(lang_set)})."
106
+ )
107
+
108
+ # Convert dictionary into tuples, splitting out cases where there are
109
+ # multiple translations for a single language.
110
+ translation_tuples = []
111
+ for lang, text in translation_dict.items():
112
+ if isinstance(text, str):
113
+ translation_tuples.append((lang, text))
114
+ else:
115
+ translation_tuples.extend([(lang, el) for el in text])
116
+
117
+ # Ensure translations are in ascending order by language code.
118
+ languages, translations = zip(*sorted(translation_tuples))
119
+
120
+ return {"language": languages, "translation": translations}
121
+
122
+ def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]:
123
+ """Flatten the TranslationVariableLanguages feature into a dictionary."""
124
+ from .features import Sequence, Value
125
+
126
+ return {
127
+ "language": Sequence(Value("string")),
128
+ "translation": Sequence(Value("string")),
129
+ }
lib/python3.12/site-packages/datasets/features/video.py ADDED
@@ -0,0 +1,280 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from dataclasses import dataclass, field
3
+ from typing import TYPE_CHECKING, Any, ClassVar, Optional, TypedDict, Union
4
+
5
+ import numpy as np
6
+ import pyarrow as pa
7
+
8
+ from .. import config
9
+ from ..download.download_config import DownloadConfig
10
+ from ..table import array_cast
11
+ from ..utils.file_utils import is_local_path, xopen
12
+ from ..utils.py_utils import string_to_dict
13
+
14
+
15
+ if TYPE_CHECKING:
16
+ from torchvision.io import VideoReader
17
+
18
+ from .features import FeatureType
19
+
20
+
21
+ class Example(TypedDict):
22
+ path: Optional[str]
23
+ bytes: Optional[bytes]
24
+
25
+
26
+ @dataclass
27
+ class Video:
28
+ """
29
+ **Experimental.** Video [`Feature`] to read video data from a video file.
30
+
31
+ Input: The Video feature accepts as input:
32
+ - A `str`: Absolute path to the video file (i.e. random access is allowed).
33
+ - A `dict` with the keys:
34
+
35
+ - `path`: String with relative path of the video file in a dataset repository.
36
+ - `bytes`: Bytes of the video file.
37
+
38
+ This is useful for archived files with sequential access.
39
+
40
+ - A `torchvision.io.VideoReader`: torchvision video reader object.
41
+
42
+ Args:
43
+ mode (`str`, *optional*):
44
+ The mode to convert the video to. If `None`, the native mode of the video is used.
45
+ decode (`bool`, defaults to `True`):
46
+ Whether to decode the video data. If `False`,
47
+ returns the underlying dictionary in the format `{"path": video_path, "bytes": video_bytes}`.
48
+
49
+ Examples:
50
+
51
+ ```py
52
+ >>> from datasets import Dataset, Video
53
+ >>> ds = Dataset.from_dict({"video":["path/to/Screen Recording.mov"]}).cast_column("video", Video())
54
+ >>> ds.features["video"]
55
+ Video(decode=True, id=None)
56
+ >>> ds[0]["video"]
57
+ <torchvision.io.video_reader.VideoReader object at 0x325b1aae0>
58
+ >>> ds = ds.cast_column('video', Video(decode=False))
59
+ {'bytes': None,
60
+ 'path': 'path/to/Screen Recording.mov'}
61
+ ```
62
+ """
63
+
64
+ decode: bool = True
65
+ id: Optional[str] = None
66
+ # Automatically constructed
67
+ dtype: ClassVar[str] = "torchvision.io.VideoReader"
68
+ pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()})
69
+ _type: str = field(default="Video", init=False, repr=False)
70
+
71
+ def __call__(self):
72
+ return self.pa_type
73
+
74
+ def encode_example(self, value: Union[str, bytes, bytearray, Example, np.ndarray, "VideoReader"]) -> Example:
75
+ """Encode example into a format for Arrow.
76
+
77
+ Args:
78
+ value (`str`, `np.ndarray`, `VideoReader` or `dict`):
79
+ Data passed as input to Video feature.
80
+
81
+ Returns:
82
+ `dict` with "path" and "bytes" fields
83
+ """
84
+ if config.TORCHVISION_AVAILABLE:
85
+ from torchvision.io import VideoReader
86
+
87
+ else:
88
+ VideoReader = None
89
+
90
+ if isinstance(value, list):
91
+ value = np.array(value)
92
+
93
+ if isinstance(value, str):
94
+ return {"path": value, "bytes": None}
95
+ elif isinstance(value, (bytes, bytearray)):
96
+ return {"path": None, "bytes": value}
97
+ elif isinstance(value, np.ndarray):
98
+ # convert the video array to bytes
99
+ return encode_np_array(value)
100
+ elif VideoReader is not None and isinstance(value, VideoReader):
101
+ # convert the torchvision video reader to bytes
102
+ return encode_torchvision_video(value)
103
+ elif isinstance(value, dict):
104
+ path, bytes_ = value.get("path"), value.get("bytes")
105
+ if path is not None and os.path.isfile(path):
106
+ # we set "bytes": None to not duplicate the data if they're already available locally
107
+ return {"bytes": None, "path": path}
108
+ elif bytes_ is not None or path is not None:
109
+ # store the video bytes, and path is used to infer the video format using the file extension
110
+ return {"bytes": bytes_, "path": path}
111
+ else:
112
+ raise ValueError(
113
+ f"A video sample should have one of 'path' or 'bytes' but they are missing or None in {value}."
114
+ )
115
+ else:
116
+ raise TypeError(f"Unsupported encode_example type: {type(value)}")
117
+
118
+ def decode_example(
119
+ self,
120
+ value: Union[str, Example],
121
+ token_per_repo_id: Optional[dict[str, Union[bool, str]]] = None,
122
+ ) -> "VideoReader":
123
+ """Decode example video file into video data.
124
+
125
+ Args:
126
+ value (`str` or `dict`):
127
+ A string with the absolute video file path, a dictionary with
128
+ keys:
129
+
130
+ - `path`: String with absolute or relative video file path.
131
+ - `bytes`: The bytes of the video file.
132
+ token_per_repo_id (`dict`, *optional*):
133
+ To access and decode
134
+ video files from private repositories on the Hub, you can pass
135
+ a dictionary repo_id (`str`) -> token (`bool` or `str`).
136
+
137
+ Returns:
138
+ `torchvision.io.VideoReader`
139
+ """
140
+ if not self.decode:
141
+ raise RuntimeError("Decoding is disabled for this feature. Please use Video(decode=True) instead.")
142
+
143
+ if config.TORCHVISION_AVAILABLE:
144
+ from torchvision.io import VideoReader
145
+
146
+ else:
147
+ raise ImportError("To support decoding videos, please install 'torchvision'.")
148
+
149
+ if token_per_repo_id is None:
150
+ token_per_repo_id = {}
151
+
152
+ if isinstance(value, str):
153
+ path, bytes_ = value, None
154
+ else:
155
+ path, bytes_ = value["path"], value["bytes"]
156
+
157
+ if bytes_ is None:
158
+ if path is None:
159
+ raise ValueError(f"A video should have one of 'path' or 'bytes' but both are None in {value}.")
160
+ elif is_local_path(path):
161
+ video = VideoReader(path)
162
+ else:
163
+ video = hf_video_reader(path, token_per_repo_id=token_per_repo_id)
164
+ else:
165
+ video = VideoReader(bytes_)
166
+ video._hf_encoded = {"path": path, "bytes": bytes_}
167
+ return video
168
+
169
+ def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]:
170
+ """If in the decodable state, return the feature itself, otherwise flatten the feature into a dictionary."""
171
+ from .features import Value
172
+
173
+ return (
174
+ self
175
+ if self.decode
176
+ else {
177
+ "bytes": Value("binary"),
178
+ "path": Value("string"),
179
+ }
180
+ )
181
+
182
+ def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray, pa.ListArray]) -> pa.StructArray:
183
+ """Cast an Arrow array to the Video arrow storage type.
184
+ The Arrow types that can be converted to the Video pyarrow storage type are:
185
+
186
+ - `pa.string()` - it must contain the "path" data
187
+ - `pa.binary()` - it must contain the video bytes
188
+ - `pa.struct({"bytes": pa.binary()})`
189
+ - `pa.struct({"path": pa.string()})`
190
+ - `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter
191
+ - `pa.list(*)` - it must contain the video array data
192
+
193
+ Args:
194
+ storage (`Union[pa.StringArray, pa.StructArray, pa.ListArray]`):
195
+ PyArrow array to cast.
196
+
197
+ Returns:
198
+ `pa.StructArray`: Array in the Video arrow storage type, that is
199
+ `pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
200
+ """
201
+ if pa.types.is_string(storage.type):
202
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
203
+ storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null())
204
+ elif pa.types.is_binary(storage.type):
205
+ path_array = pa.array([None] * len(storage), type=pa.string())
206
+ storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null())
207
+ elif pa.types.is_struct(storage.type):
208
+ if storage.type.get_field_index("bytes") >= 0:
209
+ bytes_array = storage.field("bytes")
210
+ else:
211
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
212
+ if storage.type.get_field_index("path") >= 0:
213
+ path_array = storage.field("path")
214
+ else:
215
+ path_array = pa.array([None] * len(storage), type=pa.string())
216
+ storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null())
217
+ elif pa.types.is_list(storage.type):
218
+ bytes_array = pa.array(
219
+ [encode_np_array(np.array(arr))["bytes"] if arr is not None else None for arr in storage.to_pylist()],
220
+ type=pa.binary(),
221
+ )
222
+ path_array = pa.array([None] * len(storage), type=pa.string())
223
+ storage = pa.StructArray.from_arrays(
224
+ [bytes_array, path_array], ["bytes", "path"], mask=bytes_array.is_null()
225
+ )
226
+ return array_cast(storage, self.pa_type)
227
+
228
+
229
+ def video_to_bytes(video: "VideoReader") -> bytes:
230
+ """Convert a torchvision Video object to bytes using native compression if possible"""
231
+ raise NotImplementedError()
232
+
233
+
234
+ def encode_torchvision_video(video: "VideoReader") -> Example:
235
+ if hasattr(video, "_hf_encoded"):
236
+ return video._hf_encoded
237
+ else:
238
+ raise NotImplementedError(
239
+ "Encoding a VideoReader that doesn't come from datasets.Video.decode() is not implemented"
240
+ )
241
+
242
+
243
+ def encode_np_array(array: np.ndarray) -> Example:
244
+ raise NotImplementedError()
245
+
246
+
247
+ # Patching torchvision a little bit to:
248
+ # 1. store the encoded video data {"path": ..., "bytes": ...} in `video._hf_encoded``
249
+ # 2. add support for hf:// files
250
+ # This doesn't affect the normal usage of torchvision.
251
+
252
+
253
+ def hf_video_reader(
254
+ path: str, token_per_repo_id: Optional[dict[str, Union[bool, str]]] = None, stream: str = "video"
255
+ ) -> "VideoReader":
256
+ import av
257
+ from torchvision import get_video_backend
258
+ from torchvision.io import VideoReader
259
+
260
+ # Load the file from HF
261
+ if token_per_repo_id is None:
262
+ token_per_repo_id = {}
263
+ source_url = path.split("::")[-1]
264
+ pattern = config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL
265
+ source_url_fields = string_to_dict(source_url, pattern)
266
+ token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None
267
+ download_config = DownloadConfig(token=token)
268
+ f = xopen(path, "rb", download_config=download_config)
269
+
270
+ # Instantiate the VideoReader
271
+ vr = object.__new__(VideoReader)
272
+ vr.backend = get_video_backend()
273
+ if vr.backend != "pyav":
274
+ raise RuntimeError(f"Unsupported video backend for VideoReader from HF files: {vr.backend}")
275
+ vr.container = av.open(f, metadata_errors="ignore")
276
+ stream_type = stream.split(":")[0]
277
+ stream_id = 0 if len(stream.split(":")) == 1 else int(stream.split(":")[1])
278
+ vr.pyav_stream = {stream_type: stream_id}
279
+ vr._c = vr.container.decode(**vr.pyav_stream)
280
+ return vr
lib/python3.12/site-packages/datasets/filesystems/__init__.py ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import importlib
2
+ import shutil
3
+ import warnings
4
+ from typing import List
5
+
6
+ import fsspec
7
+ import fsspec.asyn
8
+ from fsspec.implementations.local import LocalFileSystem
9
+
10
+ from . import compression
11
+
12
+
13
+ COMPRESSION_FILESYSTEMS: list[compression.BaseCompressedFileFileSystem] = [
14
+ compression.Bz2FileSystem,
15
+ compression.GzipFileSystem,
16
+ compression.Lz4FileSystem,
17
+ compression.XzFileSystem,
18
+ compression.ZstdFileSystem,
19
+ ]
20
+
21
+ # Register custom filesystems
22
+ for fs_class in COMPRESSION_FILESYSTEMS:
23
+ if fs_class.protocol in fsspec.registry and fsspec.registry[fs_class.protocol] is not fs_class:
24
+ warnings.warn(f"A filesystem protocol was already set for {fs_class.protocol} and will be overwritten.")
25
+ fsspec.register_implementation(fs_class.protocol, fs_class, clobber=True)
26
+
27
+
28
+ def is_remote_filesystem(fs: fsspec.AbstractFileSystem) -> bool:
29
+ """
30
+ Checks if `fs` is a remote filesystem.
31
+
32
+ Args:
33
+ fs (`fsspec.spec.AbstractFileSystem`):
34
+ An abstract super-class for pythonic file-systems, e.g. `fsspec.filesystem(\'file\')` or `s3fs.S3FileSystem`.
35
+ """
36
+ return not isinstance(fs, LocalFileSystem)
37
+
38
+
39
+ def rename(fs: fsspec.AbstractFileSystem, src: str, dst: str):
40
+ """
41
+ Renames the file `src` in `fs` to `dst`.
42
+ """
43
+ if not is_remote_filesystem(fs):
44
+ # LocalFileSystem.mv does copy + rm, it is more efficient to simply move a local directory
45
+ shutil.move(fs._strip_protocol(src), fs._strip_protocol(dst))
46
+ else:
47
+ fs.mv(src, dst, recursive=True)
lib/python3.12/site-packages/datasets/filesystems/__pycache__/__init__.cpython-312.pyc ADDED
Binary file (2.35 kB). View file
 
lib/python3.12/site-packages/datasets/filesystems/__pycache__/compression.cpython-312.pyc ADDED
Binary file (6.04 kB). View file
 
lib/python3.12/site-packages/datasets/filesystems/compression.py ADDED
@@ -0,0 +1,127 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from functools import partial
3
+ from typing import Optional
4
+
5
+ import fsspec
6
+ from fsspec.archive import AbstractArchiveFileSystem
7
+
8
+
9
+ class BaseCompressedFileFileSystem(AbstractArchiveFileSystem):
10
+ """Read contents of compressed file as a filesystem with one file inside."""
11
+
12
+ root_marker = ""
13
+ protocol: str = (
14
+ None # protocol passed in prefix to the url. ex: "gzip", for gzip://file.txt::http://foo.bar/file.txt.gz
15
+ )
16
+ compression: str = None # compression type in fsspec. ex: "gzip"
17
+ extension: str = None # extension of the filename to strip. ex: "".gz" to get file.txt from file.txt.gz
18
+
19
+ def __init__(
20
+ self, fo: str = "", target_protocol: Optional[str] = None, target_options: Optional[dict] = None, **kwargs
21
+ ):
22
+ """
23
+ The compressed file system can be instantiated from any compressed file.
24
+ It reads the contents of compressed file as a filesystem with one file inside, as if it was an archive.
25
+
26
+ The single file inside the filesystem is named after the compresssed file,
27
+ without the compression extension at the end of the filename.
28
+
29
+ Args:
30
+ fo (:obj:``str``): Path to compressed file. Will fetch file using ``fsspec.open()``
31
+ mode (:obj:``str``): Currently, only 'rb' accepted
32
+ target_protocol(:obj:``str``, optional): To override the FS protocol inferred from a URL.
33
+ target_options (:obj:``dict``, optional): Kwargs passed when instantiating the target FS.
34
+ """
35
+ super().__init__(self, **kwargs)
36
+ self.fo = fo.__fspath__() if hasattr(fo, "__fspath__") else fo
37
+ # always open as "rb" since fsspec can then use the TextIOWrapper to make it work for "r" mode
38
+ self._open_with_fsspec = partial(
39
+ fsspec.open,
40
+ self.fo,
41
+ mode="rb",
42
+ protocol=target_protocol,
43
+ compression=self.compression,
44
+ client_kwargs={
45
+ "requote_redirect_url": False, # see https://github.com/huggingface/datasets/pull/5459
46
+ "trust_env": True, # Enable reading proxy env variables.
47
+ **(target_options or {}).pop("client_kwargs", {}), # To avoid issues if it was already passed.
48
+ },
49
+ **(target_options or {}),
50
+ )
51
+ self.compressed_name = os.path.basename(self.fo.split("::")[0])
52
+ self.uncompressed_name = (
53
+ self.compressed_name[: self.compressed_name.rindex(".")]
54
+ if "." in self.compressed_name
55
+ else self.compressed_name
56
+ )
57
+ self.dir_cache = None
58
+
59
+ @classmethod
60
+ def _strip_protocol(cls, path):
61
+ # compressed file paths are always relative to the archive root
62
+ return super()._strip_protocol(path).lstrip("/")
63
+
64
+ def _get_dirs(self):
65
+ if self.dir_cache is None:
66
+ f = {**self._open_with_fsspec().fs.info(self.fo), "name": self.uncompressed_name}
67
+ self.dir_cache = {f["name"]: f}
68
+
69
+ def cat(self, path: str):
70
+ with self._open_with_fsspec().open() as f:
71
+ return f.read()
72
+
73
+ def _open(
74
+ self,
75
+ path: str,
76
+ mode: str = "rb",
77
+ block_size=None,
78
+ autocommit=True,
79
+ cache_options=None,
80
+ **kwargs,
81
+ ):
82
+ path = self._strip_protocol(path)
83
+ if mode != "rb":
84
+ raise ValueError(f"Tried to read with mode {mode} on file {self.fo} opened with mode 'rb'")
85
+ return self._open_with_fsspec().open()
86
+
87
+
88
+ class Bz2FileSystem(BaseCompressedFileFileSystem):
89
+ """Read contents of BZ2 file as a filesystem with one file inside."""
90
+
91
+ protocol = "bz2"
92
+ compression = "bz2"
93
+ extension = ".bz2"
94
+
95
+
96
+ class GzipFileSystem(BaseCompressedFileFileSystem):
97
+ """Read contents of GZIP file as a filesystem with one file inside."""
98
+
99
+ protocol = "gzip"
100
+ compression = "gzip"
101
+ extension = ".gz"
102
+
103
+
104
+ class Lz4FileSystem(BaseCompressedFileFileSystem):
105
+ """Read contents of LZ4 file as a filesystem with one file inside."""
106
+
107
+ protocol = "lz4"
108
+ compression = "lz4"
109
+ extension = ".lz4"
110
+
111
+
112
+ class XzFileSystem(BaseCompressedFileFileSystem):
113
+ """Read contents of .xz (LZMA) file as a filesystem with one file inside."""
114
+
115
+ protocol = "xz"
116
+ compression = "xz"
117
+ extension = ".xz"
118
+
119
+
120
+ class ZstdFileSystem(BaseCompressedFileFileSystem):
121
+ """
122
+ Read contents of .zstd file as a filesystem with one file inside.
123
+ """
124
+
125
+ protocol = "zstd"
126
+ compression = "zstd"
127
+ extension = ".zst"