Add files using upload-large-folder tool
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/INSTALLER +1 -0
- lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/METADATA +112 -0
- lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/RECORD +9 -0
- lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/WHEEL +5 -0
- lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/licenses/LICENSE +201 -0
- lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/top_level.txt +1 -0
- lib/python3.12/site-packages/datasets/__pycache__/__init__.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/__pycache__/arrow_reader.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/__pycache__/arrow_writer.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/__pycache__/builder.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/__pycache__/combine.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/__pycache__/config.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/__pycache__/data_files.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/__pycache__/distributed.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/__pycache__/exceptions.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/__pycache__/fingerprint.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/__pycache__/hub.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/__pycache__/info.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/__pycache__/inspect.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/__pycache__/keyhash.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/__pycache__/load.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/__pycache__/naming.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/__pycache__/search.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/__pycache__/splits.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/__pycache__/streaming.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/download/__init__.py +10 -0
- lib/python3.12/site-packages/datasets/download/__pycache__/__init__.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/download/__pycache__/download_config.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/download/__pycache__/download_manager.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/download/__pycache__/streaming_download_manager.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/download/download_config.py +81 -0
- lib/python3.12/site-packages/datasets/download/download_manager.py +340 -0
- lib/python3.12/site-packages/datasets/download/streaming_download_manager.py +219 -0
- lib/python3.12/site-packages/datasets/features/__init__.py +23 -0
- lib/python3.12/site-packages/datasets/features/__pycache__/__init__.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/features/__pycache__/audio.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/features/__pycache__/image.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/features/__pycache__/pdf.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/features/__pycache__/translation.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/features/__pycache__/video.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/features/audio.py +274 -0
- lib/python3.12/site-packages/datasets/features/features.py +2332 -0
- lib/python3.12/site-packages/datasets/features/image.py +382 -0
- lib/python3.12/site-packages/datasets/features/pdf.py +238 -0
- lib/python3.12/site-packages/datasets/features/translation.py +129 -0
- lib/python3.12/site-packages/datasets/features/video.py +280 -0
- lib/python3.12/site-packages/datasets/filesystems/__init__.py +47 -0
- lib/python3.12/site-packages/datasets/filesystems/__pycache__/__init__.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/filesystems/__pycache__/compression.cpython-312.pyc +0 -0
- lib/python3.12/site-packages/datasets/filesystems/compression.py +127 -0
lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/INSTALLER
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
pip
|
lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/METADATA
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Metadata-Version: 2.4
|
| 2 |
+
Name: aiosignal
|
| 3 |
+
Version: 1.4.0
|
| 4 |
+
Summary: aiosignal: a list of registered asynchronous callbacks
|
| 5 |
+
Home-page: https://github.com/aio-libs/aiosignal
|
| 6 |
+
Maintainer: aiohttp team <team@aiohttp.org>
|
| 7 |
+
Maintainer-email: team@aiohttp.org
|
| 8 |
+
License: Apache 2.0
|
| 9 |
+
Project-URL: Chat: Gitter, https://gitter.im/aio-libs/Lobby
|
| 10 |
+
Project-URL: CI: GitHub Actions, https://github.com/aio-libs/aiosignal/actions
|
| 11 |
+
Project-URL: Coverage: codecov, https://codecov.io/github/aio-libs/aiosignal
|
| 12 |
+
Project-URL: Docs: RTD, https://docs.aiosignal.org
|
| 13 |
+
Project-URL: GitHub: issues, https://github.com/aio-libs/aiosignal/issues
|
| 14 |
+
Project-URL: GitHub: repo, https://github.com/aio-libs/aiosignal
|
| 15 |
+
Classifier: License :: OSI Approved :: Apache Software License
|
| 16 |
+
Classifier: Intended Audience :: Developers
|
| 17 |
+
Classifier: Programming Language :: Python
|
| 18 |
+
Classifier: Programming Language :: Python :: 3
|
| 19 |
+
Classifier: Programming Language :: Python :: 3 :: Only
|
| 20 |
+
Classifier: Development Status :: 5 - Production/Stable
|
| 21 |
+
Classifier: Operating System :: POSIX
|
| 22 |
+
Classifier: Operating System :: MacOS :: MacOS X
|
| 23 |
+
Classifier: Operating System :: Microsoft :: Windows
|
| 24 |
+
Classifier: Framework :: AsyncIO
|
| 25 |
+
Requires-Python: >=3.9
|
| 26 |
+
Description-Content-Type: text/x-rst
|
| 27 |
+
License-File: LICENSE
|
| 28 |
+
Requires-Dist: frozenlist>=1.1.0
|
| 29 |
+
Requires-Dist: typing-extensions>=4.2; python_version < "3.13"
|
| 30 |
+
Dynamic: license-file
|
| 31 |
+
|
| 32 |
+
=========
|
| 33 |
+
aiosignal
|
| 34 |
+
=========
|
| 35 |
+
|
| 36 |
+
.. image:: https://github.com/aio-libs/aiosignal/workflows/CI/badge.svg
|
| 37 |
+
:target: https://github.com/aio-libs/aiosignal/actions?query=workflow%3ACI
|
| 38 |
+
:alt: GitHub status for master branch
|
| 39 |
+
|
| 40 |
+
.. image:: https://codecov.io/gh/aio-libs/aiosignal/branch/master/graph/badge.svg?flag=pytest
|
| 41 |
+
:target: https://codecov.io/gh/aio-libs/aiosignal?flags[0]=pytest
|
| 42 |
+
:alt: codecov.io status for master branch
|
| 43 |
+
|
| 44 |
+
.. image:: https://badge.fury.io/py/aiosignal.svg
|
| 45 |
+
:target: https://pypi.org/project/aiosignal
|
| 46 |
+
:alt: Latest PyPI package version
|
| 47 |
+
|
| 48 |
+
.. image:: https://readthedocs.org/projects/aiosignal/badge/?version=latest
|
| 49 |
+
:target: https://aiosignal.readthedocs.io/
|
| 50 |
+
:alt: Latest Read The Docs
|
| 51 |
+
|
| 52 |
+
.. image:: https://img.shields.io/discourse/topics?server=https%3A%2F%2Faio-libs.discourse.group%2F
|
| 53 |
+
:target: https://aio-libs.discourse.group/
|
| 54 |
+
:alt: Discourse group for io-libs
|
| 55 |
+
|
| 56 |
+
.. image:: https://badges.gitter.im/Join%20Chat.svg
|
| 57 |
+
:target: https://gitter.im/aio-libs/Lobby
|
| 58 |
+
:alt: Chat on Gitter
|
| 59 |
+
|
| 60 |
+
Introduction
|
| 61 |
+
============
|
| 62 |
+
|
| 63 |
+
A project to manage callbacks in `asyncio` projects.
|
| 64 |
+
|
| 65 |
+
``Signal`` is a list of registered asynchronous callbacks.
|
| 66 |
+
|
| 67 |
+
The signal's life-cycle has two stages: after creation its content
|
| 68 |
+
could be filled by using standard list operations: ``sig.append()``
|
| 69 |
+
etc.
|
| 70 |
+
|
| 71 |
+
After you call ``sig.freeze()`` the signal is *frozen*: adding, removing
|
| 72 |
+
and dropping callbacks is forbidden.
|
| 73 |
+
|
| 74 |
+
The only available operation is calling the previously registered
|
| 75 |
+
callbacks by using ``await sig.send(data)``.
|
| 76 |
+
|
| 77 |
+
For concrete usage examples see the `Signals
|
| 78 |
+
<https://docs.aiohttp.org/en/stable/web_advanced.html#aiohttp-web-signals>
|
| 79 |
+
section of the `Web Server Advanced
|
| 80 |
+
<https://docs.aiohttp.org/en/stable/web_advanced.html>` chapter of the `aiohttp
|
| 81 |
+
documentation`_.
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
Installation
|
| 85 |
+
------------
|
| 86 |
+
|
| 87 |
+
::
|
| 88 |
+
|
| 89 |
+
$ pip install aiosignal
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
Documentation
|
| 93 |
+
=============
|
| 94 |
+
|
| 95 |
+
https://aiosignal.readthedocs.io/
|
| 96 |
+
|
| 97 |
+
License
|
| 98 |
+
=======
|
| 99 |
+
|
| 100 |
+
``aiosignal`` is offered under the Apache 2 license.
|
| 101 |
+
|
| 102 |
+
Source code
|
| 103 |
+
===========
|
| 104 |
+
|
| 105 |
+
The project is hosted on GitHub_
|
| 106 |
+
|
| 107 |
+
Please file an issue in the `bug tracker
|
| 108 |
+
<https://github.com/aio-libs/aiosignal/issues>`_ if you have found a bug
|
| 109 |
+
or have some suggestions to improve the library.
|
| 110 |
+
|
| 111 |
+
.. _GitHub: https://github.com/aio-libs/aiosignal
|
| 112 |
+
.. _aiohttp documentation: https://docs.aiohttp.org/
|
lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/RECORD
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
aiosignal-1.4.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
|
| 2 |
+
aiosignal-1.4.0.dist-info/METADATA,sha256=CSR-8dqLxpZyjUcTDnAuQwf299EB1sSFv_nzpxznAI0,3662
|
| 3 |
+
aiosignal-1.4.0.dist-info/RECORD,,
|
| 4 |
+
aiosignal-1.4.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
| 5 |
+
aiosignal-1.4.0.dist-info/licenses/LICENSE,sha256=b9UkPpLdf5jsacesN3co50kFcJ_1J6W_mNbQJjwE9bY,11332
|
| 6 |
+
aiosignal-1.4.0.dist-info/top_level.txt,sha256=z45aNOKGDdrI1roqZY3BGXQ22kJFPHBmVdwtLYLtXC0,10
|
| 7 |
+
aiosignal/__init__.py,sha256=TIkmUG9HTBt4dfq2nISYBiZiRB2xwvFtEZydLP0HPL4,1537
|
| 8 |
+
aiosignal/__pycache__/__init__.cpython-312.pyc,,
|
| 9 |
+
aiosignal/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/WHEEL
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Wheel-Version: 1.0
|
| 2 |
+
Generator: setuptools (80.9.0)
|
| 3 |
+
Root-Is-Purelib: true
|
| 4 |
+
Tag: py3-none-any
|
| 5 |
+
|
lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/licenses/LICENSE
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Apache License
|
| 2 |
+
Version 2.0, January 2004
|
| 3 |
+
http://www.apache.org/licenses/
|
| 4 |
+
|
| 5 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 6 |
+
|
| 7 |
+
1. Definitions.
|
| 8 |
+
|
| 9 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 10 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 11 |
+
|
| 12 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 13 |
+
the copyright owner that is granting the License.
|
| 14 |
+
|
| 15 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 16 |
+
other entities that control, are controlled by, or are under common
|
| 17 |
+
control with that entity. For the purposes of this definition,
|
| 18 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 19 |
+
direction or management of such entity, whether by contract or
|
| 20 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 21 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 22 |
+
|
| 23 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 24 |
+
exercising permissions granted by this License.
|
| 25 |
+
|
| 26 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 27 |
+
including but not limited to software source code, documentation
|
| 28 |
+
source, and configuration files.
|
| 29 |
+
|
| 30 |
+
"Object" form shall mean any form resulting from mechanical
|
| 31 |
+
transformation or translation of a Source form, including but
|
| 32 |
+
not limited to compiled object code, generated documentation,
|
| 33 |
+
and conversions to other media types.
|
| 34 |
+
|
| 35 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 36 |
+
Object form, made available under the License, as indicated by a
|
| 37 |
+
copyright notice that is included in or attached to the work
|
| 38 |
+
(an example is provided in the Appendix below).
|
| 39 |
+
|
| 40 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 41 |
+
form, that is based on (or derived from) the Work and for which the
|
| 42 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 43 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 44 |
+
of this License, Derivative Works shall not include works that remain
|
| 45 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 46 |
+
the Work and Derivative Works thereof.
|
| 47 |
+
|
| 48 |
+
"Contribution" shall mean any work of authorship, including
|
| 49 |
+
the original version of the Work and any modifications or additions
|
| 50 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 51 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 52 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 53 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 54 |
+
means any form of electronic, verbal, or written communication sent
|
| 55 |
+
to the Licensor or its representatives, including but not limited to
|
| 56 |
+
communication on electronic mailing lists, source code control systems,
|
| 57 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 58 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 59 |
+
excluding communication that is conspicuously marked or otherwise
|
| 60 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 61 |
+
|
| 62 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 63 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 64 |
+
subsequently incorporated within the Work.
|
| 65 |
+
|
| 66 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 67 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 68 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 69 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 70 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 71 |
+
Work and such Derivative Works in Source or Object form.
|
| 72 |
+
|
| 73 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 74 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 75 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 76 |
+
(except as stated in this section) patent license to make, have made,
|
| 77 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 78 |
+
where such license applies only to those patent claims licensable
|
| 79 |
+
by such Contributor that are necessarily infringed by their
|
| 80 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 81 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 82 |
+
institute patent litigation against any entity (including a
|
| 83 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 84 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 85 |
+
or contributory patent infringement, then any patent licenses
|
| 86 |
+
granted to You under this License for that Work shall terminate
|
| 87 |
+
as of the date such litigation is filed.
|
| 88 |
+
|
| 89 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 90 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 91 |
+
modifications, and in Source or Object form, provided that You
|
| 92 |
+
meet the following conditions:
|
| 93 |
+
|
| 94 |
+
(a) You must give any other recipients of the Work or
|
| 95 |
+
Derivative Works a copy of this License; and
|
| 96 |
+
|
| 97 |
+
(b) You must cause any modified files to carry prominent notices
|
| 98 |
+
stating that You changed the files; and
|
| 99 |
+
|
| 100 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 101 |
+
that You distribute, all copyright, patent, trademark, and
|
| 102 |
+
attribution notices from the Source form of the Work,
|
| 103 |
+
excluding those notices that do not pertain to any part of
|
| 104 |
+
the Derivative Works; and
|
| 105 |
+
|
| 106 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 107 |
+
distribution, then any Derivative Works that You distribute must
|
| 108 |
+
include a readable copy of the attribution notices contained
|
| 109 |
+
within such NOTICE file, excluding those notices that do not
|
| 110 |
+
pertain to any part of the Derivative Works, in at least one
|
| 111 |
+
of the following places: within a NOTICE text file distributed
|
| 112 |
+
as part of the Derivative Works; within the Source form or
|
| 113 |
+
documentation, if provided along with the Derivative Works; or,
|
| 114 |
+
within a display generated by the Derivative Works, if and
|
| 115 |
+
wherever such third-party notices normally appear. The contents
|
| 116 |
+
of the NOTICE file are for informational purposes only and
|
| 117 |
+
do not modify the License. You may add Your own attribution
|
| 118 |
+
notices within Derivative Works that You distribute, alongside
|
| 119 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 120 |
+
that such additional attribution notices cannot be construed
|
| 121 |
+
as modifying the License.
|
| 122 |
+
|
| 123 |
+
You may add Your own copyright statement to Your modifications and
|
| 124 |
+
may provide additional or different license terms and conditions
|
| 125 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 126 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 127 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 128 |
+
the conditions stated in this License.
|
| 129 |
+
|
| 130 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 131 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 132 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 133 |
+
this License, without any additional terms or conditions.
|
| 134 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 135 |
+
the terms of any separate license agreement you may have executed
|
| 136 |
+
with Licensor regarding such Contributions.
|
| 137 |
+
|
| 138 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 139 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 140 |
+
except as required for reasonable and customary use in describing the
|
| 141 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 142 |
+
|
| 143 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 144 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 145 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 146 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 147 |
+
implied, including, without limitation, any warranties or conditions
|
| 148 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 149 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 150 |
+
appropriateness of using or redistributing the Work and assume any
|
| 151 |
+
risks associated with Your exercise of permissions under this License.
|
| 152 |
+
|
| 153 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 154 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 155 |
+
unless required by applicable law (such as deliberate and grossly
|
| 156 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 157 |
+
liable to You for damages, including any direct, indirect, special,
|
| 158 |
+
incidental, or consequential damages of any character arising as a
|
| 159 |
+
result of this License or out of the use or inability to use the
|
| 160 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 161 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 162 |
+
other commercial damages or losses), even if such Contributor
|
| 163 |
+
has been advised of the possibility of such damages.
|
| 164 |
+
|
| 165 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 166 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 167 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 168 |
+
or other liability obligations and/or rights consistent with this
|
| 169 |
+
License. However, in accepting such obligations, You may act only
|
| 170 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 171 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 172 |
+
defend, and hold each Contributor harmless for any liability
|
| 173 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 174 |
+
of your accepting any such warranty or additional liability.
|
| 175 |
+
|
| 176 |
+
END OF TERMS AND CONDITIONS
|
| 177 |
+
|
| 178 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 179 |
+
|
| 180 |
+
To apply the Apache License to your work, attach the following
|
| 181 |
+
boilerplate notice, with the fields enclosed by brackets "{}"
|
| 182 |
+
replaced with your own identifying information. (Don't include
|
| 183 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 184 |
+
comment syntax for the file format. We also recommend that a
|
| 185 |
+
file or class name and description of purpose be included on the
|
| 186 |
+
same "printed page" as the copyright notice for easier
|
| 187 |
+
identification within third-party archives.
|
| 188 |
+
|
| 189 |
+
Copyright 2013-2019 Nikolay Kim and Andrew Svetlov
|
| 190 |
+
|
| 191 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 192 |
+
you may not use this file except in compliance with the License.
|
| 193 |
+
You may obtain a copy of the License at
|
| 194 |
+
|
| 195 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 196 |
+
|
| 197 |
+
Unless required by applicable law or agreed to in writing, software
|
| 198 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 199 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 200 |
+
See the License for the specific language governing permissions and
|
| 201 |
+
limitations under the License.
|
lib/python3.12/site-packages/aiosignal-1.4.0.dist-info/top_level.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
aiosignal
|
lib/python3.12/site-packages/datasets/__pycache__/__init__.cpython-312.pyc
ADDED
|
Binary file (1.45 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/__pycache__/arrow_reader.cpython-312.pyc
ADDED
|
Binary file (28.4 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/__pycache__/arrow_writer.cpython-312.pyc
ADDED
|
Binary file (34.9 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/__pycache__/builder.cpython-312.pyc
ADDED
|
Binary file (97 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/__pycache__/combine.cpython-312.pyc
ADDED
|
Binary file (10.8 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/__pycache__/config.cpython-312.pyc
ADDED
|
Binary file (11.9 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/__pycache__/data_files.cpython-312.pyc
ADDED
|
Binary file (37.1 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/__pycache__/distributed.cpython-312.pyc
ADDED
|
Binary file (1.85 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/__pycache__/exceptions.cpython-312.pyc
ADDED
|
Binary file (6.65 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/__pycache__/fingerprint.cpython-312.pyc
ADDED
|
Binary file (21.6 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/__pycache__/hub.cpython-312.pyc
ADDED
|
Binary file (9.9 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/__pycache__/info.cpython-312.pyc
ADDED
|
Binary file (24.5 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/__pycache__/inspect.cpython-312.pyc
ADDED
|
Binary file (17 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/__pycache__/keyhash.cpython-312.pyc
ADDED
|
Binary file (4.71 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/__pycache__/load.cpython-312.pyc
ADDED
|
Binary file (99.6 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/__pycache__/naming.cpython-312.pyc
ADDED
|
Binary file (3.95 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/__pycache__/search.cpython-312.pyc
ADDED
|
Binary file (42.8 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/__pycache__/splits.cpython-312.pyc
ADDED
|
Binary file (29.4 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/__pycache__/streaming.cpython-312.pyc
ADDED
|
Binary file (7.8 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/download/__init__.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__all__ = [
|
| 2 |
+
"DownloadConfig",
|
| 3 |
+
"DownloadManager",
|
| 4 |
+
"DownloadMode",
|
| 5 |
+
"StreamingDownloadManager",
|
| 6 |
+
]
|
| 7 |
+
|
| 8 |
+
from .download_config import DownloadConfig
|
| 9 |
+
from .download_manager import DownloadManager, DownloadMode
|
| 10 |
+
from .streaming_download_manager import StreamingDownloadManager
|
lib/python3.12/site-packages/datasets/download/__pycache__/__init__.cpython-312.pyc
ADDED
|
Binary file (457 Bytes). View file
|
|
|
lib/python3.12/site-packages/datasets/download/__pycache__/download_config.cpython-312.pyc
ADDED
|
Binary file (5.1 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/download/__pycache__/download_manager.cpython-312.pyc
ADDED
|
Binary file (15.1 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/download/__pycache__/streaming_download_manager.cpython-312.pyc
ADDED
|
Binary file (9.31 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/download/download_config.py
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import copy
|
| 2 |
+
from dataclasses import dataclass, field
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
from typing import Any, Optional, Union
|
| 5 |
+
|
| 6 |
+
from .. import config
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
@dataclass
|
| 10 |
+
class DownloadConfig:
|
| 11 |
+
"""Configuration for our cached path manager.
|
| 12 |
+
|
| 13 |
+
Attributes:
|
| 14 |
+
cache_dir (`str` or `Path`, *optional*):
|
| 15 |
+
Specify a cache directory to save the file to (overwrite the
|
| 16 |
+
default cache dir).
|
| 17 |
+
force_download (`bool`, defaults to `False`):
|
| 18 |
+
If `True`, re-download the file even if it's already cached in
|
| 19 |
+
the cache dir.
|
| 20 |
+
resume_download (`bool`, defaults to `False`):
|
| 21 |
+
If `True`, resume the download if an incompletely received file is
|
| 22 |
+
found.
|
| 23 |
+
proxies (`dict`, *optional*):
|
| 24 |
+
user_agent (`str`, *optional*):
|
| 25 |
+
Optional string or dict that will be appended to the user-agent on remote
|
| 26 |
+
requests.
|
| 27 |
+
extract_compressed_file (`bool`, defaults to `False`):
|
| 28 |
+
If `True` and the path point to a zip or tar file,
|
| 29 |
+
extract the compressed file in a folder along the archive.
|
| 30 |
+
force_extract (`bool`, defaults to `False`):
|
| 31 |
+
If `True` when `extract_compressed_file` is `True` and the archive
|
| 32 |
+
was already extracted, re-extract the archive and override the folder where it was extracted.
|
| 33 |
+
delete_extracted (`bool`, defaults to `False`):
|
| 34 |
+
Whether to delete (or keep) the extracted files.
|
| 35 |
+
extract_on_the_fly (`bool`, defaults to `False`):
|
| 36 |
+
If `True`, extract compressed files while they are being read.
|
| 37 |
+
use_etag (`bool`, defaults to `True`):
|
| 38 |
+
Whether to use the ETag HTTP response header to validate the cached files.
|
| 39 |
+
num_proc (`int`, *optional*):
|
| 40 |
+
The number of processes to launch to download the files in parallel.
|
| 41 |
+
max_retries (`int`, default to `1`):
|
| 42 |
+
The number of times to retry an HTTP request if it fails.
|
| 43 |
+
token (`str` or `bool`, *optional*):
|
| 44 |
+
Optional string or boolean to use as Bearer token
|
| 45 |
+
for remote files on the Datasets Hub. If `True`, or not specified, will get token from `~/.huggingface`.
|
| 46 |
+
storage_options (`dict`, *optional*):
|
| 47 |
+
Key/value pairs to be passed on to the dataset file-system backend, if any.
|
| 48 |
+
download_desc (`str`, *optional*):
|
| 49 |
+
A description to be displayed alongside with the progress bar while downloading the files.
|
| 50 |
+
disable_tqdm (`bool`, defaults to `False`):
|
| 51 |
+
Whether to disable the individual files download progress bar
|
| 52 |
+
"""
|
| 53 |
+
|
| 54 |
+
cache_dir: Optional[Union[str, Path]] = None
|
| 55 |
+
force_download: bool = False
|
| 56 |
+
resume_download: bool = False
|
| 57 |
+
local_files_only: bool = False
|
| 58 |
+
proxies: Optional[dict] = None
|
| 59 |
+
user_agent: Optional[str] = None
|
| 60 |
+
extract_compressed_file: bool = False
|
| 61 |
+
force_extract: bool = False
|
| 62 |
+
delete_extracted: bool = False
|
| 63 |
+
extract_on_the_fly: bool = False
|
| 64 |
+
use_etag: bool = True
|
| 65 |
+
num_proc: Optional[int] = None
|
| 66 |
+
max_retries: int = 1
|
| 67 |
+
token: Optional[Union[str, bool]] = None
|
| 68 |
+
storage_options: dict[str, Any] = field(default_factory=dict)
|
| 69 |
+
download_desc: Optional[str] = None
|
| 70 |
+
disable_tqdm: bool = False
|
| 71 |
+
|
| 72 |
+
def copy(self) -> "DownloadConfig":
|
| 73 |
+
return self.__class__(**{k: copy.deepcopy(v) for k, v in self.__dict__.items()})
|
| 74 |
+
|
| 75 |
+
def __setattr__(self, name, value):
|
| 76 |
+
if name == "token" and getattr(self, "storage_options", None) is not None:
|
| 77 |
+
if "hf" not in self.storage_options:
|
| 78 |
+
self.storage_options["hf"] = {"token": value, "endpoint": config.HF_ENDPOINT}
|
| 79 |
+
elif getattr(self.storage_options["hf"], "token", None) is None:
|
| 80 |
+
self.storage_options["hf"]["token"] = value
|
| 81 |
+
super().__setattr__(name, value)
|
lib/python3.12/site-packages/datasets/download/download_manager.py
ADDED
|
@@ -0,0 +1,340 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2020 The TensorFlow Datasets Authors.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
|
| 15 |
+
# Lint as: python3
|
| 16 |
+
"""Download manager interface."""
|
| 17 |
+
|
| 18 |
+
import enum
|
| 19 |
+
import io
|
| 20 |
+
import multiprocessing
|
| 21 |
+
import os
|
| 22 |
+
from datetime import datetime
|
| 23 |
+
from functools import partial
|
| 24 |
+
from typing import Optional, Union
|
| 25 |
+
|
| 26 |
+
import fsspec
|
| 27 |
+
from fsspec.core import url_to_fs
|
| 28 |
+
from tqdm.contrib.concurrent import thread_map
|
| 29 |
+
|
| 30 |
+
from .. import config
|
| 31 |
+
from ..utils import tqdm as hf_tqdm
|
| 32 |
+
from ..utils.file_utils import (
|
| 33 |
+
ArchiveIterable,
|
| 34 |
+
FilesIterable,
|
| 35 |
+
cached_path,
|
| 36 |
+
is_relative_path,
|
| 37 |
+
stack_multiprocessing_download_progress_bars,
|
| 38 |
+
url_or_path_join,
|
| 39 |
+
)
|
| 40 |
+
from ..utils.info_utils import get_size_checksum_dict
|
| 41 |
+
from ..utils.logging import get_logger, tqdm
|
| 42 |
+
from ..utils.py_utils import NestedDataStructure, map_nested
|
| 43 |
+
from ..utils.track import tracked_str
|
| 44 |
+
from .download_config import DownloadConfig
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
logger = get_logger(__name__)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
class DownloadMode(enum.Enum):
|
| 51 |
+
"""`Enum` for how to treat pre-existing downloads and data.
|
| 52 |
+
|
| 53 |
+
The default mode is `REUSE_DATASET_IF_EXISTS`, which will reuse both
|
| 54 |
+
raw downloads and the prepared dataset if they exist.
|
| 55 |
+
|
| 56 |
+
The generations modes:
|
| 57 |
+
|
| 58 |
+
| | Downloads | Dataset |
|
| 59 |
+
|-------------------------------------|-----------|---------|
|
| 60 |
+
| `REUSE_DATASET_IF_EXISTS` (default) | Reuse | Reuse |
|
| 61 |
+
| `REUSE_CACHE_IF_EXISTS` | Reuse | Fresh |
|
| 62 |
+
| `FORCE_REDOWNLOAD` | Fresh | Fresh |
|
| 63 |
+
|
| 64 |
+
"""
|
| 65 |
+
|
| 66 |
+
REUSE_DATASET_IF_EXISTS = "reuse_dataset_if_exists"
|
| 67 |
+
REUSE_CACHE_IF_EXISTS = "reuse_cache_if_exists"
|
| 68 |
+
FORCE_REDOWNLOAD = "force_redownload"
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
class DownloadManager:
|
| 72 |
+
is_streaming = False
|
| 73 |
+
|
| 74 |
+
def __init__(
|
| 75 |
+
self,
|
| 76 |
+
dataset_name: Optional[str] = None,
|
| 77 |
+
data_dir: Optional[str] = None,
|
| 78 |
+
download_config: Optional[DownloadConfig] = None,
|
| 79 |
+
base_path: Optional[str] = None,
|
| 80 |
+
record_checksums=True,
|
| 81 |
+
):
|
| 82 |
+
"""Download manager constructor.
|
| 83 |
+
|
| 84 |
+
Args:
|
| 85 |
+
data_dir:
|
| 86 |
+
can be used to specify a manual directory to get the files from.
|
| 87 |
+
dataset_name (`str`):
|
| 88 |
+
name of dataset this instance will be used for. If
|
| 89 |
+
provided, downloads will contain which datasets they were used for.
|
| 90 |
+
download_config (`DownloadConfig`):
|
| 91 |
+
to specify the cache directory and other
|
| 92 |
+
download options
|
| 93 |
+
base_path (`str`):
|
| 94 |
+
base path that is used when relative paths are used to
|
| 95 |
+
download files. This can be a remote url.
|
| 96 |
+
record_checksums (`bool`, defaults to `True`):
|
| 97 |
+
Whether to record the checksums of the downloaded files. If None, the value is inferred from the builder.
|
| 98 |
+
"""
|
| 99 |
+
self._dataset_name = dataset_name
|
| 100 |
+
self._data_dir = data_dir
|
| 101 |
+
self._base_path = base_path or os.path.abspath(".")
|
| 102 |
+
# To record what is being used: {url: {num_bytes: int, checksum: str}}
|
| 103 |
+
self._recorded_sizes_checksums: dict[str, dict[str, Optional[Union[int, str]]]] = {}
|
| 104 |
+
self.record_checksums = record_checksums
|
| 105 |
+
self.download_config = download_config or DownloadConfig()
|
| 106 |
+
self.downloaded_paths = {}
|
| 107 |
+
self.extracted_paths = {}
|
| 108 |
+
|
| 109 |
+
@property
|
| 110 |
+
def manual_dir(self):
|
| 111 |
+
return self._data_dir
|
| 112 |
+
|
| 113 |
+
@property
|
| 114 |
+
def downloaded_size(self):
|
| 115 |
+
"""Returns the total size of downloaded files."""
|
| 116 |
+
return sum(checksums_dict["num_bytes"] for checksums_dict in self._recorded_sizes_checksums.values())
|
| 117 |
+
|
| 118 |
+
def _record_sizes_checksums(self, url_or_urls: NestedDataStructure, downloaded_path_or_paths: NestedDataStructure):
|
| 119 |
+
"""Record size/checksum of downloaded files."""
|
| 120 |
+
delay = 5
|
| 121 |
+
for url, path in hf_tqdm(
|
| 122 |
+
list(zip(url_or_urls.flatten(), downloaded_path_or_paths.flatten())),
|
| 123 |
+
delay=delay,
|
| 124 |
+
desc="Computing checksums",
|
| 125 |
+
):
|
| 126 |
+
# call str to support PathLike objects
|
| 127 |
+
self._recorded_sizes_checksums[str(url)] = get_size_checksum_dict(
|
| 128 |
+
path, record_checksum=self.record_checksums
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
def download(self, url_or_urls):
|
| 132 |
+
"""Download given URL(s).
|
| 133 |
+
|
| 134 |
+
By default, only one process is used for download. Pass customized `download_config.num_proc` to change this behavior.
|
| 135 |
+
|
| 136 |
+
Args:
|
| 137 |
+
url_or_urls (`str` or `list` or `dict`):
|
| 138 |
+
URL or `list` or `dict` of URLs to download. Each URL is a `str`.
|
| 139 |
+
|
| 140 |
+
Returns:
|
| 141 |
+
`str` or `list` or `dict`:
|
| 142 |
+
The downloaded paths matching the given input `url_or_urls`.
|
| 143 |
+
|
| 144 |
+
Example:
|
| 145 |
+
|
| 146 |
+
```py
|
| 147 |
+
>>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz')
|
| 148 |
+
```
|
| 149 |
+
"""
|
| 150 |
+
download_config = self.download_config.copy()
|
| 151 |
+
download_config.extract_compressed_file = False
|
| 152 |
+
if download_config.download_desc is None:
|
| 153 |
+
download_config.download_desc = "Downloading data"
|
| 154 |
+
|
| 155 |
+
download_func = partial(self._download_batched, download_config=download_config)
|
| 156 |
+
|
| 157 |
+
start_time = datetime.now()
|
| 158 |
+
with stack_multiprocessing_download_progress_bars():
|
| 159 |
+
downloaded_path_or_paths = map_nested(
|
| 160 |
+
download_func,
|
| 161 |
+
url_or_urls,
|
| 162 |
+
map_tuple=True,
|
| 163 |
+
num_proc=download_config.num_proc,
|
| 164 |
+
desc="Downloading data files",
|
| 165 |
+
batched=True,
|
| 166 |
+
batch_size=-1,
|
| 167 |
+
)
|
| 168 |
+
duration = datetime.now() - start_time
|
| 169 |
+
logger.info(f"Downloading took {duration.total_seconds() // 60} min")
|
| 170 |
+
url_or_urls = NestedDataStructure(url_or_urls)
|
| 171 |
+
downloaded_path_or_paths = NestedDataStructure(downloaded_path_or_paths)
|
| 172 |
+
self.downloaded_paths.update(dict(zip(url_or_urls.flatten(), downloaded_path_or_paths.flatten())))
|
| 173 |
+
|
| 174 |
+
start_time = datetime.now()
|
| 175 |
+
self._record_sizes_checksums(url_or_urls, downloaded_path_or_paths)
|
| 176 |
+
duration = datetime.now() - start_time
|
| 177 |
+
logger.info(f"Checksum Computation took {duration.total_seconds() // 60} min")
|
| 178 |
+
|
| 179 |
+
return downloaded_path_or_paths.data
|
| 180 |
+
|
| 181 |
+
def _download_batched(
|
| 182 |
+
self,
|
| 183 |
+
url_or_filenames: list[str],
|
| 184 |
+
download_config: DownloadConfig,
|
| 185 |
+
) -> list[str]:
|
| 186 |
+
if len(url_or_filenames) >= 16:
|
| 187 |
+
download_config = download_config.copy()
|
| 188 |
+
download_config.disable_tqdm = True
|
| 189 |
+
download_func = partial(self._download_single, download_config=download_config)
|
| 190 |
+
|
| 191 |
+
fs: fsspec.AbstractFileSystem
|
| 192 |
+
path = str(url_or_filenames[0])
|
| 193 |
+
if is_relative_path(path):
|
| 194 |
+
# append the relative path to the base_path
|
| 195 |
+
path = url_or_path_join(self._base_path, path)
|
| 196 |
+
fs, path = url_to_fs(path, **download_config.storage_options)
|
| 197 |
+
size = 0
|
| 198 |
+
try:
|
| 199 |
+
size = fs.info(path).get("size", 0)
|
| 200 |
+
except Exception:
|
| 201 |
+
pass
|
| 202 |
+
max_workers = (
|
| 203 |
+
config.HF_DATASETS_MULTITHREADING_MAX_WORKERS if size < (20 << 20) else 1
|
| 204 |
+
) # enable multithreading if files are small
|
| 205 |
+
|
| 206 |
+
return thread_map(
|
| 207 |
+
download_func,
|
| 208 |
+
url_or_filenames,
|
| 209 |
+
desc=download_config.download_desc or "Downloading",
|
| 210 |
+
unit="files",
|
| 211 |
+
position=multiprocessing.current_process()._identity[-1] # contains the ranks of subprocesses
|
| 212 |
+
if os.environ.get("HF_DATASETS_STACK_MULTIPROCESSING_DOWNLOAD_PROGRESS_BARS") == "1"
|
| 213 |
+
and multiprocessing.current_process()._identity
|
| 214 |
+
else None,
|
| 215 |
+
max_workers=max_workers,
|
| 216 |
+
tqdm_class=tqdm,
|
| 217 |
+
)
|
| 218 |
+
else:
|
| 219 |
+
return [
|
| 220 |
+
self._download_single(url_or_filename, download_config=download_config)
|
| 221 |
+
for url_or_filename in url_or_filenames
|
| 222 |
+
]
|
| 223 |
+
|
| 224 |
+
def _download_single(self, url_or_filename: str, download_config: DownloadConfig) -> str:
|
| 225 |
+
url_or_filename = str(url_or_filename)
|
| 226 |
+
if is_relative_path(url_or_filename):
|
| 227 |
+
# append the relative path to the base_path
|
| 228 |
+
url_or_filename = url_or_path_join(self._base_path, url_or_filename)
|
| 229 |
+
out = cached_path(url_or_filename, download_config=download_config)
|
| 230 |
+
out = tracked_str(out)
|
| 231 |
+
out.set_origin(url_or_filename)
|
| 232 |
+
return out
|
| 233 |
+
|
| 234 |
+
def iter_archive(self, path_or_buf: Union[str, io.BufferedReader]):
|
| 235 |
+
"""Iterate over files within an archive.
|
| 236 |
+
|
| 237 |
+
Args:
|
| 238 |
+
path_or_buf (`str` or `io.BufferedReader`):
|
| 239 |
+
Archive path or archive binary file object.
|
| 240 |
+
|
| 241 |
+
Yields:
|
| 242 |
+
`tuple[str, io.BufferedReader]`:
|
| 243 |
+
2-tuple (path_within_archive, file_object).
|
| 244 |
+
File object is opened in binary mode.
|
| 245 |
+
|
| 246 |
+
Example:
|
| 247 |
+
|
| 248 |
+
```py
|
| 249 |
+
>>> archive = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz')
|
| 250 |
+
>>> files = dl_manager.iter_archive(archive)
|
| 251 |
+
```
|
| 252 |
+
"""
|
| 253 |
+
|
| 254 |
+
if hasattr(path_or_buf, "read"):
|
| 255 |
+
return ArchiveIterable.from_buf(path_or_buf)
|
| 256 |
+
else:
|
| 257 |
+
return ArchiveIterable.from_urlpath(path_or_buf)
|
| 258 |
+
|
| 259 |
+
def iter_files(self, paths: Union[str, list[str]]):
|
| 260 |
+
"""Iterate over file paths.
|
| 261 |
+
|
| 262 |
+
Args:
|
| 263 |
+
paths (`str` or `list` of `str`):
|
| 264 |
+
Root paths.
|
| 265 |
+
|
| 266 |
+
Yields:
|
| 267 |
+
`str`: File path.
|
| 268 |
+
|
| 269 |
+
Example:
|
| 270 |
+
|
| 271 |
+
```py
|
| 272 |
+
>>> files = dl_manager.download_and_extract('https://huggingface.co/datasets/beans/resolve/main/data/train.zip')
|
| 273 |
+
>>> files = dl_manager.iter_files(files)
|
| 274 |
+
```
|
| 275 |
+
"""
|
| 276 |
+
return FilesIterable.from_urlpaths(paths)
|
| 277 |
+
|
| 278 |
+
def extract(self, path_or_paths):
|
| 279 |
+
"""Extract given path(s).
|
| 280 |
+
|
| 281 |
+
Args:
|
| 282 |
+
path_or_paths (path or `list` or `dict`):
|
| 283 |
+
Path of file to extract. Each path is a `str`.
|
| 284 |
+
|
| 285 |
+
Returns:
|
| 286 |
+
extracted_path(s): `str`, The extracted paths matching the given input
|
| 287 |
+
path_or_paths.
|
| 288 |
+
|
| 289 |
+
Example:
|
| 290 |
+
|
| 291 |
+
```py
|
| 292 |
+
>>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz')
|
| 293 |
+
>>> extracted_files = dl_manager.extract(downloaded_files)
|
| 294 |
+
```
|
| 295 |
+
"""
|
| 296 |
+
download_config = self.download_config.copy()
|
| 297 |
+
download_config.extract_compressed_file = True
|
| 298 |
+
extract_func = partial(self._download_single, download_config=download_config)
|
| 299 |
+
extracted_paths = map_nested(
|
| 300 |
+
extract_func,
|
| 301 |
+
path_or_paths,
|
| 302 |
+
num_proc=download_config.num_proc,
|
| 303 |
+
desc="Extracting data files",
|
| 304 |
+
)
|
| 305 |
+
path_or_paths = NestedDataStructure(path_or_paths)
|
| 306 |
+
extracted_paths = NestedDataStructure(extracted_paths)
|
| 307 |
+
self.extracted_paths.update(dict(zip(path_or_paths.flatten(), extracted_paths.flatten())))
|
| 308 |
+
return extracted_paths.data
|
| 309 |
+
|
| 310 |
+
def download_and_extract(self, url_or_urls):
|
| 311 |
+
"""Download and extract given `url_or_urls`.
|
| 312 |
+
|
| 313 |
+
Is roughly equivalent to:
|
| 314 |
+
|
| 315 |
+
```
|
| 316 |
+
extracted_paths = dl_manager.extract(dl_manager.download(url_or_urls))
|
| 317 |
+
```
|
| 318 |
+
|
| 319 |
+
Args:
|
| 320 |
+
url_or_urls (`str` or `list` or `dict`):
|
| 321 |
+
URL or `list` or `dict` of URLs to download and extract. Each URL is a `str`.
|
| 322 |
+
|
| 323 |
+
Returns:
|
| 324 |
+
extracted_path(s): `str`, extracted paths of given URL(s).
|
| 325 |
+
"""
|
| 326 |
+
return self.extract(self.download(url_or_urls))
|
| 327 |
+
|
| 328 |
+
def get_recorded_sizes_checksums(self):
|
| 329 |
+
return self._recorded_sizes_checksums.copy()
|
| 330 |
+
|
| 331 |
+
def delete_extracted_files(self):
|
| 332 |
+
paths_to_delete = set(self.extracted_paths.values()) - set(self.downloaded_paths.values())
|
| 333 |
+
for key, path in list(self.extracted_paths.items()):
|
| 334 |
+
if path in paths_to_delete and os.path.isfile(path):
|
| 335 |
+
os.remove(path)
|
| 336 |
+
del self.extracted_paths[key]
|
| 337 |
+
|
| 338 |
+
def manage_extracted_files(self):
|
| 339 |
+
if self.download_config.delete_extracted:
|
| 340 |
+
self.delete_extracted_files()
|
lib/python3.12/site-packages/datasets/download/streaming_download_manager.py
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import io
|
| 2 |
+
import os
|
| 3 |
+
from collections.abc import Iterable
|
| 4 |
+
from typing import Optional, Union
|
| 5 |
+
|
| 6 |
+
from ..utils.file_utils import ( # noqa: F401 # backward compatibility
|
| 7 |
+
SINGLE_FILE_COMPRESSION_PROTOCOLS,
|
| 8 |
+
ArchiveIterable,
|
| 9 |
+
FilesIterable,
|
| 10 |
+
_get_extraction_protocol,
|
| 11 |
+
_get_path_extension,
|
| 12 |
+
_prepare_path_and_storage_options,
|
| 13 |
+
is_relative_path,
|
| 14 |
+
url_or_path_join,
|
| 15 |
+
xbasename,
|
| 16 |
+
xdirname,
|
| 17 |
+
xet_parse,
|
| 18 |
+
xexists,
|
| 19 |
+
xgetsize,
|
| 20 |
+
xglob,
|
| 21 |
+
xgzip_open,
|
| 22 |
+
xisdir,
|
| 23 |
+
xisfile,
|
| 24 |
+
xjoin,
|
| 25 |
+
xlistdir,
|
| 26 |
+
xnumpy_load,
|
| 27 |
+
xopen,
|
| 28 |
+
xpandas_read_csv,
|
| 29 |
+
xpandas_read_excel,
|
| 30 |
+
xPath,
|
| 31 |
+
xpyarrow_parquet_read_table,
|
| 32 |
+
xrelpath,
|
| 33 |
+
xsio_loadmat,
|
| 34 |
+
xsplit,
|
| 35 |
+
xsplitext,
|
| 36 |
+
xwalk,
|
| 37 |
+
xxml_dom_minidom_parse,
|
| 38 |
+
)
|
| 39 |
+
from ..utils.logging import get_logger
|
| 40 |
+
from ..utils.py_utils import map_nested
|
| 41 |
+
from .download_config import DownloadConfig
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
logger = get_logger(__name__)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
class StreamingDownloadManager:
|
| 48 |
+
"""
|
| 49 |
+
Download manager that uses the "::" separator to navigate through (possibly remote) compressed archives.
|
| 50 |
+
Contrary to the regular `DownloadManager`, the `download` and `extract` methods don't actually download nor extract
|
| 51 |
+
data, but they rather return the path or url that could be opened using the `xopen` function which extends the
|
| 52 |
+
built-in `open` function to stream data from remote files.
|
| 53 |
+
"""
|
| 54 |
+
|
| 55 |
+
is_streaming = True
|
| 56 |
+
|
| 57 |
+
def __init__(
|
| 58 |
+
self,
|
| 59 |
+
dataset_name: Optional[str] = None,
|
| 60 |
+
data_dir: Optional[str] = None,
|
| 61 |
+
download_config: Optional[DownloadConfig] = None,
|
| 62 |
+
base_path: Optional[str] = None,
|
| 63 |
+
):
|
| 64 |
+
self._dataset_name = dataset_name
|
| 65 |
+
self._data_dir = data_dir
|
| 66 |
+
self._base_path = base_path or os.path.abspath(".")
|
| 67 |
+
self.download_config = download_config or DownloadConfig()
|
| 68 |
+
self.downloaded_size = None
|
| 69 |
+
self.record_checksums = False
|
| 70 |
+
|
| 71 |
+
@property
|
| 72 |
+
def manual_dir(self):
|
| 73 |
+
return self._data_dir
|
| 74 |
+
|
| 75 |
+
def download(self, url_or_urls):
|
| 76 |
+
"""Normalize URL(s) of files to stream data from.
|
| 77 |
+
This is the lazy version of `DownloadManager.download` for streaming.
|
| 78 |
+
|
| 79 |
+
Args:
|
| 80 |
+
url_or_urls (`str` or `list` or `dict`):
|
| 81 |
+
URL(s) of files to stream data from. Each url is a `str`.
|
| 82 |
+
|
| 83 |
+
Returns:
|
| 84 |
+
url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input url_or_urls.
|
| 85 |
+
|
| 86 |
+
Example:
|
| 87 |
+
|
| 88 |
+
```py
|
| 89 |
+
>>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz')
|
| 90 |
+
```
|
| 91 |
+
"""
|
| 92 |
+
url_or_urls = map_nested(self._download_single, url_or_urls, map_tuple=True)
|
| 93 |
+
return url_or_urls
|
| 94 |
+
|
| 95 |
+
def _download_single(self, urlpath: str) -> str:
|
| 96 |
+
urlpath = str(urlpath)
|
| 97 |
+
if is_relative_path(urlpath):
|
| 98 |
+
# append the relative path to the base_path
|
| 99 |
+
urlpath = url_or_path_join(self._base_path, urlpath)
|
| 100 |
+
return urlpath
|
| 101 |
+
|
| 102 |
+
def extract(self, url_or_urls):
|
| 103 |
+
"""Add extraction protocol for given url(s) for streaming.
|
| 104 |
+
|
| 105 |
+
This is the lazy version of `DownloadManager.extract` for streaming.
|
| 106 |
+
|
| 107 |
+
Args:
|
| 108 |
+
url_or_urls (`str` or `list` or `dict`):
|
| 109 |
+
URL(s) of files to stream data from. Each url is a `str`.
|
| 110 |
+
|
| 111 |
+
Returns:
|
| 112 |
+
url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input `url_or_urls`.
|
| 113 |
+
|
| 114 |
+
Example:
|
| 115 |
+
|
| 116 |
+
```py
|
| 117 |
+
>>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz')
|
| 118 |
+
>>> extracted_files = dl_manager.extract(downloaded_files)
|
| 119 |
+
```
|
| 120 |
+
"""
|
| 121 |
+
urlpaths = map_nested(self._extract, url_or_urls, map_tuple=True)
|
| 122 |
+
return urlpaths
|
| 123 |
+
|
| 124 |
+
def _extract(self, urlpath: str) -> str:
|
| 125 |
+
urlpath = str(urlpath)
|
| 126 |
+
protocol = _get_extraction_protocol(urlpath, download_config=self.download_config)
|
| 127 |
+
# get inner file: zip://train-00000.json.gz::https://foo.bar/data.zip -> zip://train-00000.json.gz
|
| 128 |
+
path = urlpath.split("::")[0]
|
| 129 |
+
extension = _get_path_extension(path)
|
| 130 |
+
if extension in ["tgz", "tar"] or path.endswith((".tar.gz", ".tar.bz2", ".tar.xz")):
|
| 131 |
+
raise NotImplementedError(
|
| 132 |
+
f"Extraction protocol for TAR archives like '{urlpath}' is not implemented in streaming mode. "
|
| 133 |
+
f"Please use `dl_manager.iter_archive` instead.\n\n"
|
| 134 |
+
f"Example usage:\n\n"
|
| 135 |
+
f"\turl = dl_manager.download(url)\n"
|
| 136 |
+
f"\ttar_archive_iterator = dl_manager.iter_archive(url)\n\n"
|
| 137 |
+
f"\tfor filename, file in tar_archive_iterator:\n"
|
| 138 |
+
f"\t\t..."
|
| 139 |
+
)
|
| 140 |
+
if protocol is None:
|
| 141 |
+
# no extraction
|
| 142 |
+
return urlpath
|
| 143 |
+
elif protocol in SINGLE_FILE_COMPRESSION_PROTOCOLS:
|
| 144 |
+
# there is one single file which is the uncompressed file
|
| 145 |
+
inner_file = os.path.basename(urlpath.split("::")[0])
|
| 146 |
+
inner_file = inner_file[: inner_file.rindex(".")] if "." in inner_file else inner_file
|
| 147 |
+
return f"{protocol}://{inner_file}::{urlpath}"
|
| 148 |
+
else:
|
| 149 |
+
return f"{protocol}://::{urlpath}"
|
| 150 |
+
|
| 151 |
+
def download_and_extract(self, url_or_urls):
|
| 152 |
+
"""Prepare given `url_or_urls` for streaming (add extraction protocol).
|
| 153 |
+
|
| 154 |
+
This is the lazy version of `DownloadManager.download_and_extract` for streaming.
|
| 155 |
+
|
| 156 |
+
Is equivalent to:
|
| 157 |
+
|
| 158 |
+
```
|
| 159 |
+
urls = dl_manager.extract(dl_manager.download(url_or_urls))
|
| 160 |
+
```
|
| 161 |
+
|
| 162 |
+
Args:
|
| 163 |
+
url_or_urls (`str` or `list` or `dict`):
|
| 164 |
+
URL(s) to stream from data from. Each url is a `str`.
|
| 165 |
+
|
| 166 |
+
Returns:
|
| 167 |
+
url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input `url_or_urls`.
|
| 168 |
+
"""
|
| 169 |
+
return self.extract(self.download(url_or_urls))
|
| 170 |
+
|
| 171 |
+
def iter_archive(self, urlpath_or_buf: Union[str, io.BufferedReader]) -> Iterable[tuple]:
|
| 172 |
+
"""Iterate over files within an archive.
|
| 173 |
+
|
| 174 |
+
Args:
|
| 175 |
+
urlpath_or_buf (`str` or `io.BufferedReader`):
|
| 176 |
+
Archive path or archive binary file object.
|
| 177 |
+
|
| 178 |
+
Yields:
|
| 179 |
+
`tuple[str, io.BufferedReader]`:
|
| 180 |
+
2-tuple (path_within_archive, file_object).
|
| 181 |
+
File object is opened in binary mode.
|
| 182 |
+
|
| 183 |
+
Example:
|
| 184 |
+
|
| 185 |
+
```py
|
| 186 |
+
>>> archive = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz')
|
| 187 |
+
>>> files = dl_manager.iter_archive(archive)
|
| 188 |
+
```
|
| 189 |
+
"""
|
| 190 |
+
|
| 191 |
+
if hasattr(urlpath_or_buf, "read"):
|
| 192 |
+
return ArchiveIterable.from_buf(urlpath_or_buf)
|
| 193 |
+
else:
|
| 194 |
+
return ArchiveIterable.from_urlpath(urlpath_or_buf, download_config=self.download_config)
|
| 195 |
+
|
| 196 |
+
def iter_files(self, urlpaths: Union[str, list[str]]) -> Iterable[str]:
|
| 197 |
+
"""Iterate over files.
|
| 198 |
+
|
| 199 |
+
Args:
|
| 200 |
+
urlpaths (`str` or `list` of `str`):
|
| 201 |
+
Root paths.
|
| 202 |
+
|
| 203 |
+
Yields:
|
| 204 |
+
str: File URL path.
|
| 205 |
+
|
| 206 |
+
Example:
|
| 207 |
+
|
| 208 |
+
```py
|
| 209 |
+
>>> files = dl_manager.download_and_extract('https://huggingface.co/datasets/beans/resolve/main/data/train.zip')
|
| 210 |
+
>>> files = dl_manager.iter_files(files)
|
| 211 |
+
```
|
| 212 |
+
"""
|
| 213 |
+
return FilesIterable.from_urlpaths(urlpaths, download_config=self.download_config)
|
| 214 |
+
|
| 215 |
+
def manage_extracted_files(self):
|
| 216 |
+
pass
|
| 217 |
+
|
| 218 |
+
def get_recorded_sizes_checksums(self):
|
| 219 |
+
pass
|
lib/python3.12/site-packages/datasets/features/__init__.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__all__ = [
|
| 2 |
+
"Audio",
|
| 3 |
+
"Array2D",
|
| 4 |
+
"Array3D",
|
| 5 |
+
"Array4D",
|
| 6 |
+
"Array5D",
|
| 7 |
+
"ClassLabel",
|
| 8 |
+
"Features",
|
| 9 |
+
"LargeList",
|
| 10 |
+
"Sequence",
|
| 11 |
+
"Value",
|
| 12 |
+
"Image",
|
| 13 |
+
"Translation",
|
| 14 |
+
"TranslationVariableLanguages",
|
| 15 |
+
"Video",
|
| 16 |
+
"Pdf",
|
| 17 |
+
]
|
| 18 |
+
from .audio import Audio
|
| 19 |
+
from .features import Array2D, Array3D, Array4D, Array5D, ClassLabel, Features, LargeList, Sequence, Value
|
| 20 |
+
from .image import Image
|
| 21 |
+
from .pdf import Pdf
|
| 22 |
+
from .translation import Translation, TranslationVariableLanguages
|
| 23 |
+
from .video import Video
|
lib/python3.12/site-packages/datasets/features/__pycache__/__init__.cpython-312.pyc
ADDED
|
Binary file (724 Bytes). View file
|
|
|
lib/python3.12/site-packages/datasets/features/__pycache__/audio.cpython-312.pyc
ADDED
|
Binary file (15.4 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/features/__pycache__/image.cpython-312.pyc
ADDED
|
Binary file (19.9 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/features/__pycache__/pdf.cpython-312.pyc
ADDED
|
Binary file (11.5 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/features/__pycache__/translation.cpython-312.pyc
ADDED
|
Binary file (6.47 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/features/__pycache__/video.cpython-312.pyc
ADDED
|
Binary file (13.8 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/features/audio.py
ADDED
|
@@ -0,0 +1,274 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from dataclasses import dataclass, field
|
| 3 |
+
from io import BytesIO
|
| 4 |
+
from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union
|
| 5 |
+
|
| 6 |
+
import numpy as np
|
| 7 |
+
import pyarrow as pa
|
| 8 |
+
|
| 9 |
+
from .. import config
|
| 10 |
+
from ..download.download_config import DownloadConfig
|
| 11 |
+
from ..table import array_cast
|
| 12 |
+
from ..utils.file_utils import xopen, xsplitext
|
| 13 |
+
from ..utils.py_utils import no_op_if_value_is_null, string_to_dict
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
if TYPE_CHECKING:
|
| 17 |
+
from .features import FeatureType
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
@dataclass
|
| 21 |
+
class Audio:
|
| 22 |
+
"""Audio [`Feature`] to extract audio data from an audio file.
|
| 23 |
+
|
| 24 |
+
Input: The Audio feature accepts as input:
|
| 25 |
+
- A `str`: Absolute path to the audio file (i.e. random access is allowed).
|
| 26 |
+
- A `dict` with the keys:
|
| 27 |
+
|
| 28 |
+
- `path`: String with relative path of the audio file to the archive file.
|
| 29 |
+
- `bytes`: Bytes content of the audio file.
|
| 30 |
+
|
| 31 |
+
This is useful for archived files with sequential access.
|
| 32 |
+
|
| 33 |
+
- A `dict` with the keys:
|
| 34 |
+
|
| 35 |
+
- `path`: String with relative path of the audio file to the archive file.
|
| 36 |
+
- `array`: Array containing the audio sample
|
| 37 |
+
- `sampling_rate`: Integer corresponding to the sampling rate of the audio sample.
|
| 38 |
+
|
| 39 |
+
This is useful for archived files with sequential access.
|
| 40 |
+
|
| 41 |
+
Args:
|
| 42 |
+
sampling_rate (`int`, *optional*):
|
| 43 |
+
Target sampling rate. If `None`, the native sampling rate is used.
|
| 44 |
+
mono (`bool`, defaults to `True`):
|
| 45 |
+
Whether to convert the audio signal to mono by averaging samples across
|
| 46 |
+
channels.
|
| 47 |
+
decode (`bool`, defaults to `True`):
|
| 48 |
+
Whether to decode the audio data. If `False`,
|
| 49 |
+
returns the underlying dictionary in the format `{"path": audio_path, "bytes": audio_bytes}`.
|
| 50 |
+
|
| 51 |
+
Example:
|
| 52 |
+
|
| 53 |
+
```py
|
| 54 |
+
>>> from datasets import load_dataset, Audio
|
| 55 |
+
>>> ds = load_dataset("PolyAI/minds14", name="en-US", split="train")
|
| 56 |
+
>>> ds = ds.cast_column("audio", Audio(sampling_rate=16000))
|
| 57 |
+
>>> ds[0]["audio"]
|
| 58 |
+
{'array': array([ 2.3443763e-05, 2.1729663e-04, 2.2145823e-04, ...,
|
| 59 |
+
3.8356509e-05, -7.3497440e-06, -2.1754686e-05], dtype=float32),
|
| 60 |
+
'path': '/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav',
|
| 61 |
+
'sampling_rate': 16000}
|
| 62 |
+
```
|
| 63 |
+
"""
|
| 64 |
+
|
| 65 |
+
sampling_rate: Optional[int] = None
|
| 66 |
+
mono: bool = True
|
| 67 |
+
decode: bool = True
|
| 68 |
+
id: Optional[str] = None
|
| 69 |
+
# Automatically constructed
|
| 70 |
+
dtype: ClassVar[str] = "dict"
|
| 71 |
+
pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()})
|
| 72 |
+
_type: str = field(default="Audio", init=False, repr=False)
|
| 73 |
+
|
| 74 |
+
def __call__(self):
|
| 75 |
+
return self.pa_type
|
| 76 |
+
|
| 77 |
+
def encode_example(self, value: Union[str, bytes, bytearray, dict]) -> dict:
|
| 78 |
+
"""Encode example into a format for Arrow.
|
| 79 |
+
|
| 80 |
+
Args:
|
| 81 |
+
value (`str` or `dict`):
|
| 82 |
+
Data passed as input to Audio feature.
|
| 83 |
+
|
| 84 |
+
Returns:
|
| 85 |
+
`dict`
|
| 86 |
+
"""
|
| 87 |
+
try:
|
| 88 |
+
import soundfile as sf # soundfile is a dependency of librosa, needed to decode audio files.
|
| 89 |
+
except ImportError as err:
|
| 90 |
+
raise ImportError("To support encoding audio data, please install 'soundfile'.") from err
|
| 91 |
+
if isinstance(value, str):
|
| 92 |
+
return {"bytes": None, "path": value}
|
| 93 |
+
elif isinstance(value, (bytes, bytearray)):
|
| 94 |
+
return {"bytes": value, "path": None}
|
| 95 |
+
elif "array" in value:
|
| 96 |
+
# convert the audio array to wav bytes
|
| 97 |
+
buffer = BytesIO()
|
| 98 |
+
sf.write(buffer, value["array"], value["sampling_rate"], format="wav")
|
| 99 |
+
return {"bytes": buffer.getvalue(), "path": None}
|
| 100 |
+
elif value.get("path") is not None and os.path.isfile(value["path"]):
|
| 101 |
+
# we set "bytes": None to not duplicate the data if they're already available locally
|
| 102 |
+
if value["path"].endswith("pcm"):
|
| 103 |
+
# "PCM" only has raw audio bytes
|
| 104 |
+
if value.get("sampling_rate") is None:
|
| 105 |
+
# At least, If you want to convert "PCM-byte" to "WAV-byte", you have to know sampling rate
|
| 106 |
+
raise KeyError("To use PCM files, please specify a 'sampling_rate' in Audio object")
|
| 107 |
+
if value.get("bytes"):
|
| 108 |
+
# If we already had PCM-byte, we don`t have to make "read file, make bytes" (just use it!)
|
| 109 |
+
bytes_value = np.frombuffer(value["bytes"], dtype=np.int16).astype(np.float32) / 32767
|
| 110 |
+
else:
|
| 111 |
+
bytes_value = np.memmap(value["path"], dtype="h", mode="r").astype(np.float32) / 32767
|
| 112 |
+
|
| 113 |
+
buffer = BytesIO(b"")
|
| 114 |
+
sf.write(buffer, bytes_value, value["sampling_rate"], format="wav")
|
| 115 |
+
return {"bytes": buffer.getvalue(), "path": None}
|
| 116 |
+
else:
|
| 117 |
+
return {"bytes": None, "path": value.get("path")}
|
| 118 |
+
elif value.get("bytes") is not None or value.get("path") is not None:
|
| 119 |
+
# store the audio bytes, and path is used to infer the audio format using the file extension
|
| 120 |
+
return {"bytes": value.get("bytes"), "path": value.get("path")}
|
| 121 |
+
else:
|
| 122 |
+
raise ValueError(
|
| 123 |
+
f"An audio sample should have one of 'path' or 'bytes' but they are missing or None in {value}."
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
def decode_example(
|
| 127 |
+
self, value: dict, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None
|
| 128 |
+
) -> dict:
|
| 129 |
+
"""Decode example audio file into audio data.
|
| 130 |
+
|
| 131 |
+
Args:
|
| 132 |
+
value (`dict`):
|
| 133 |
+
A dictionary with keys:
|
| 134 |
+
|
| 135 |
+
- `path`: String with relative audio file path.
|
| 136 |
+
- `bytes`: Bytes of the audio file.
|
| 137 |
+
token_per_repo_id (`dict`, *optional*):
|
| 138 |
+
To access and decode
|
| 139 |
+
audio files from private repositories on the Hub, you can pass
|
| 140 |
+
a dictionary repo_id (`str`) -> token (`bool` or `str`)
|
| 141 |
+
|
| 142 |
+
Returns:
|
| 143 |
+
`dict`
|
| 144 |
+
"""
|
| 145 |
+
if not self.decode:
|
| 146 |
+
raise RuntimeError("Decoding is disabled for this feature. Please use Audio(decode=True) instead.")
|
| 147 |
+
|
| 148 |
+
path, file = (value["path"], BytesIO(value["bytes"])) if value["bytes"] is not None else (value["path"], None)
|
| 149 |
+
if path is None and file is None:
|
| 150 |
+
raise ValueError(f"An audio sample should have one of 'path' or 'bytes' but both are None in {value}.")
|
| 151 |
+
|
| 152 |
+
try:
|
| 153 |
+
import librosa
|
| 154 |
+
import soundfile as sf
|
| 155 |
+
except ImportError as err:
|
| 156 |
+
raise ImportError("To support decoding audio files, please install 'librosa' and 'soundfile'.") from err
|
| 157 |
+
|
| 158 |
+
audio_format = xsplitext(path)[1][1:].lower() if path is not None else None
|
| 159 |
+
if not config.IS_OPUS_SUPPORTED and audio_format == "opus":
|
| 160 |
+
raise RuntimeError(
|
| 161 |
+
"Decoding 'opus' files requires system library 'libsndfile'>=1.0.31, "
|
| 162 |
+
'You can try to update `soundfile` python library: `pip install "soundfile>=0.12.1"`. '
|
| 163 |
+
)
|
| 164 |
+
elif not config.IS_MP3_SUPPORTED and audio_format == "mp3":
|
| 165 |
+
raise RuntimeError(
|
| 166 |
+
"Decoding 'mp3' files requires system library 'libsndfile'>=1.1.0, "
|
| 167 |
+
'You can try to update `soundfile` python library: `pip install "soundfile>=0.12.1"`. '
|
| 168 |
+
)
|
| 169 |
+
|
| 170 |
+
if file is None:
|
| 171 |
+
token_per_repo_id = token_per_repo_id or {}
|
| 172 |
+
source_url = path.split("::")[-1]
|
| 173 |
+
pattern = (
|
| 174 |
+
config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL
|
| 175 |
+
)
|
| 176 |
+
source_url_fields = string_to_dict(source_url, pattern)
|
| 177 |
+
token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None
|
| 178 |
+
|
| 179 |
+
download_config = DownloadConfig(token=token)
|
| 180 |
+
with xopen(path, "rb", download_config=download_config) as f:
|
| 181 |
+
array, sampling_rate = sf.read(f)
|
| 182 |
+
|
| 183 |
+
else:
|
| 184 |
+
array, sampling_rate = sf.read(file)
|
| 185 |
+
|
| 186 |
+
array = array.T
|
| 187 |
+
if self.mono:
|
| 188 |
+
array = librosa.to_mono(array)
|
| 189 |
+
if self.sampling_rate and self.sampling_rate != sampling_rate:
|
| 190 |
+
array = librosa.resample(array, orig_sr=sampling_rate, target_sr=self.sampling_rate)
|
| 191 |
+
sampling_rate = self.sampling_rate
|
| 192 |
+
|
| 193 |
+
return {"path": path, "array": array, "sampling_rate": sampling_rate}
|
| 194 |
+
|
| 195 |
+
def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]:
|
| 196 |
+
"""If in the decodable state, raise an error, otherwise flatten the feature into a dictionary."""
|
| 197 |
+
from .features import Value
|
| 198 |
+
|
| 199 |
+
if self.decode:
|
| 200 |
+
raise ValueError("Cannot flatten a decoded Audio feature.")
|
| 201 |
+
return {
|
| 202 |
+
"bytes": Value("binary"),
|
| 203 |
+
"path": Value("string"),
|
| 204 |
+
}
|
| 205 |
+
|
| 206 |
+
def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray]) -> pa.StructArray:
|
| 207 |
+
"""Cast an Arrow array to the Audio arrow storage type.
|
| 208 |
+
The Arrow types that can be converted to the Audio pyarrow storage type are:
|
| 209 |
+
|
| 210 |
+
- `pa.string()` - it must contain the "path" data
|
| 211 |
+
- `pa.binary()` - it must contain the audio bytes
|
| 212 |
+
- `pa.struct({"bytes": pa.binary()})`
|
| 213 |
+
- `pa.struct({"path": pa.string()})`
|
| 214 |
+
- `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter
|
| 215 |
+
|
| 216 |
+
Args:
|
| 217 |
+
storage (`Union[pa.StringArray, pa.StructArray]`):
|
| 218 |
+
PyArrow array to cast.
|
| 219 |
+
|
| 220 |
+
Returns:
|
| 221 |
+
`pa.StructArray`: Array in the Audio arrow storage type, that is
|
| 222 |
+
`pa.struct({"bytes": pa.binary(), "path": pa.string()})`
|
| 223 |
+
"""
|
| 224 |
+
if pa.types.is_string(storage.type):
|
| 225 |
+
bytes_array = pa.array([None] * len(storage), type=pa.binary())
|
| 226 |
+
storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null())
|
| 227 |
+
elif pa.types.is_binary(storage.type):
|
| 228 |
+
path_array = pa.array([None] * len(storage), type=pa.string())
|
| 229 |
+
storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null())
|
| 230 |
+
elif pa.types.is_struct(storage.type) and storage.type.get_all_field_indices("array"):
|
| 231 |
+
storage = pa.array([Audio().encode_example(x) if x is not None else None for x in storage.to_pylist()])
|
| 232 |
+
elif pa.types.is_struct(storage.type):
|
| 233 |
+
if storage.type.get_field_index("bytes") >= 0:
|
| 234 |
+
bytes_array = storage.field("bytes")
|
| 235 |
+
else:
|
| 236 |
+
bytes_array = pa.array([None] * len(storage), type=pa.binary())
|
| 237 |
+
if storage.type.get_field_index("path") >= 0:
|
| 238 |
+
path_array = storage.field("path")
|
| 239 |
+
else:
|
| 240 |
+
path_array = pa.array([None] * len(storage), type=pa.string())
|
| 241 |
+
storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null())
|
| 242 |
+
return array_cast(storage, self.pa_type)
|
| 243 |
+
|
| 244 |
+
def embed_storage(self, storage: pa.StructArray) -> pa.StructArray:
|
| 245 |
+
"""Embed audio files into the Arrow array.
|
| 246 |
+
|
| 247 |
+
Args:
|
| 248 |
+
storage (`pa.StructArray`):
|
| 249 |
+
PyArrow array to embed.
|
| 250 |
+
|
| 251 |
+
Returns:
|
| 252 |
+
`pa.StructArray`: Array in the Audio arrow storage type, that is
|
| 253 |
+
`pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
|
| 254 |
+
"""
|
| 255 |
+
|
| 256 |
+
@no_op_if_value_is_null
|
| 257 |
+
def path_to_bytes(path):
|
| 258 |
+
with xopen(path, "rb") as f:
|
| 259 |
+
bytes_ = f.read()
|
| 260 |
+
return bytes_
|
| 261 |
+
|
| 262 |
+
bytes_array = pa.array(
|
| 263 |
+
[
|
| 264 |
+
(path_to_bytes(x["path"]) if x["bytes"] is None else x["bytes"]) if x is not None else None
|
| 265 |
+
for x in storage.to_pylist()
|
| 266 |
+
],
|
| 267 |
+
type=pa.binary(),
|
| 268 |
+
)
|
| 269 |
+
path_array = pa.array(
|
| 270 |
+
[os.path.basename(path) if path is not None else None for path in storage.field("path").to_pylist()],
|
| 271 |
+
type=pa.string(),
|
| 272 |
+
)
|
| 273 |
+
storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=bytes_array.is_null())
|
| 274 |
+
return array_cast(storage, self.pa_type)
|
lib/python3.12/site-packages/datasets/features/features.py
ADDED
|
@@ -0,0 +1,2332 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
|
| 15 |
+
# Lint as: python3
|
| 16 |
+
"""This class handle features definition in datasets and some utilities to display table type."""
|
| 17 |
+
|
| 18 |
+
import copy
|
| 19 |
+
import json
|
| 20 |
+
import re
|
| 21 |
+
import sys
|
| 22 |
+
from collections.abc import Iterable, Mapping
|
| 23 |
+
from collections.abc import Sequence as SequenceABC
|
| 24 |
+
from collections.abc import Sequence as Sequence_
|
| 25 |
+
from dataclasses import InitVar, dataclass, field, fields
|
| 26 |
+
from functools import reduce, wraps
|
| 27 |
+
from operator import mul
|
| 28 |
+
from typing import Any, Callable, ClassVar, Literal, Optional, Union
|
| 29 |
+
|
| 30 |
+
import numpy as np
|
| 31 |
+
import pandas as pd
|
| 32 |
+
import pyarrow as pa
|
| 33 |
+
import pyarrow.compute as pc
|
| 34 |
+
import pyarrow.types
|
| 35 |
+
from pandas.api.extensions import ExtensionArray as PandasExtensionArray
|
| 36 |
+
from pandas.api.extensions import ExtensionDtype as PandasExtensionDtype
|
| 37 |
+
|
| 38 |
+
from .. import config
|
| 39 |
+
from ..naming import camelcase_to_snakecase, snakecase_to_camelcase
|
| 40 |
+
from ..table import array_cast
|
| 41 |
+
from ..utils import experimental, logging
|
| 42 |
+
from ..utils.py_utils import asdict, first_non_null_value, zip_dict
|
| 43 |
+
from .audio import Audio
|
| 44 |
+
from .image import Image, encode_pil_image
|
| 45 |
+
from .pdf import Pdf, encode_pdfplumber_pdf
|
| 46 |
+
from .translation import Translation, TranslationVariableLanguages
|
| 47 |
+
from .video import Video
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
logger = logging.get_logger(__name__)
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def _arrow_to_datasets_dtype(arrow_type: pa.DataType) -> str:
|
| 54 |
+
"""
|
| 55 |
+
_arrow_to_datasets_dtype takes a pyarrow.DataType and converts it to a datasets string dtype.
|
| 56 |
+
In effect, `dt == string_to_arrow(_arrow_to_datasets_dtype(dt))`
|
| 57 |
+
"""
|
| 58 |
+
if pyarrow.types.is_null(arrow_type):
|
| 59 |
+
return "null"
|
| 60 |
+
elif pyarrow.types.is_boolean(arrow_type):
|
| 61 |
+
return "bool"
|
| 62 |
+
elif pyarrow.types.is_int8(arrow_type):
|
| 63 |
+
return "int8"
|
| 64 |
+
elif pyarrow.types.is_int16(arrow_type):
|
| 65 |
+
return "int16"
|
| 66 |
+
elif pyarrow.types.is_int32(arrow_type):
|
| 67 |
+
return "int32"
|
| 68 |
+
elif pyarrow.types.is_int64(arrow_type):
|
| 69 |
+
return "int64"
|
| 70 |
+
elif pyarrow.types.is_uint8(arrow_type):
|
| 71 |
+
return "uint8"
|
| 72 |
+
elif pyarrow.types.is_uint16(arrow_type):
|
| 73 |
+
return "uint16"
|
| 74 |
+
elif pyarrow.types.is_uint32(arrow_type):
|
| 75 |
+
return "uint32"
|
| 76 |
+
elif pyarrow.types.is_uint64(arrow_type):
|
| 77 |
+
return "uint64"
|
| 78 |
+
elif pyarrow.types.is_float16(arrow_type):
|
| 79 |
+
return "float16" # pyarrow dtype is "halffloat"
|
| 80 |
+
elif pyarrow.types.is_float32(arrow_type):
|
| 81 |
+
return "float32" # pyarrow dtype is "float"
|
| 82 |
+
elif pyarrow.types.is_float64(arrow_type):
|
| 83 |
+
return "float64" # pyarrow dtype is "double"
|
| 84 |
+
elif pyarrow.types.is_time32(arrow_type):
|
| 85 |
+
return f"time32[{pa.type_for_alias(str(arrow_type)).unit}]"
|
| 86 |
+
elif pyarrow.types.is_time64(arrow_type):
|
| 87 |
+
return f"time64[{pa.type_for_alias(str(arrow_type)).unit}]"
|
| 88 |
+
elif pyarrow.types.is_timestamp(arrow_type):
|
| 89 |
+
if arrow_type.tz is None:
|
| 90 |
+
return f"timestamp[{arrow_type.unit}]"
|
| 91 |
+
elif arrow_type.tz:
|
| 92 |
+
return f"timestamp[{arrow_type.unit}, tz={arrow_type.tz}]"
|
| 93 |
+
else:
|
| 94 |
+
raise ValueError(f"Unexpected timestamp object {arrow_type}.")
|
| 95 |
+
elif pyarrow.types.is_date32(arrow_type):
|
| 96 |
+
return "date32" # pyarrow dtype is "date32[day]"
|
| 97 |
+
elif pyarrow.types.is_date64(arrow_type):
|
| 98 |
+
return "date64" # pyarrow dtype is "date64[ms]"
|
| 99 |
+
elif pyarrow.types.is_duration(arrow_type):
|
| 100 |
+
return f"duration[{arrow_type.unit}]"
|
| 101 |
+
elif pyarrow.types.is_decimal128(arrow_type):
|
| 102 |
+
return f"decimal128({arrow_type.precision}, {arrow_type.scale})"
|
| 103 |
+
elif pyarrow.types.is_decimal256(arrow_type):
|
| 104 |
+
return f"decimal256({arrow_type.precision}, {arrow_type.scale})"
|
| 105 |
+
elif pyarrow.types.is_binary(arrow_type):
|
| 106 |
+
return "binary"
|
| 107 |
+
elif pyarrow.types.is_large_binary(arrow_type):
|
| 108 |
+
return "large_binary"
|
| 109 |
+
elif pyarrow.types.is_string(arrow_type):
|
| 110 |
+
return "string"
|
| 111 |
+
elif pyarrow.types.is_large_string(arrow_type):
|
| 112 |
+
return "large_string"
|
| 113 |
+
elif pyarrow.types.is_dictionary(arrow_type):
|
| 114 |
+
return _arrow_to_datasets_dtype(arrow_type.value_type)
|
| 115 |
+
else:
|
| 116 |
+
raise ValueError(f"Arrow type {arrow_type} does not have a datasets dtype equivalent.")
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def string_to_arrow(datasets_dtype: str) -> pa.DataType:
|
| 120 |
+
"""
|
| 121 |
+
string_to_arrow takes a datasets string dtype and converts it to a pyarrow.DataType.
|
| 122 |
+
|
| 123 |
+
In effect, `dt == string_to_arrow(_arrow_to_datasets_dtype(dt))`
|
| 124 |
+
|
| 125 |
+
This is necessary because the datasets.Value() primitive type is constructed using a string dtype
|
| 126 |
+
|
| 127 |
+
Value(dtype=str)
|
| 128 |
+
|
| 129 |
+
But Features.type (via `get_nested_type()` expects to resolve Features into a pyarrow Schema,
|
| 130 |
+
which means that each Value() must be able to resolve into a corresponding pyarrow.DataType, which is the
|
| 131 |
+
purpose of this function.
|
| 132 |
+
"""
|
| 133 |
+
|
| 134 |
+
def _dtype_error_msg(dtype, pa_dtype, examples=None, urls=None):
|
| 135 |
+
msg = f"{dtype} is not a validly formatted string representation of the pyarrow {pa_dtype} type."
|
| 136 |
+
if examples:
|
| 137 |
+
examples = ", ".join(examples[:-1]) + " or " + examples[-1] if len(examples) > 1 else examples[0]
|
| 138 |
+
msg += f"\nValid examples include: {examples}."
|
| 139 |
+
if urls:
|
| 140 |
+
urls = ", ".join(urls[:-1]) + " and " + urls[-1] if len(urls) > 1 else urls[0]
|
| 141 |
+
msg += f"\nFor more insformation, see: {urls}."
|
| 142 |
+
return msg
|
| 143 |
+
|
| 144 |
+
if datasets_dtype in pa.__dict__:
|
| 145 |
+
return pa.__dict__[datasets_dtype]()
|
| 146 |
+
|
| 147 |
+
if (datasets_dtype + "_") in pa.__dict__:
|
| 148 |
+
return pa.__dict__[datasets_dtype + "_"]()
|
| 149 |
+
|
| 150 |
+
timestamp_matches = re.search(r"^timestamp\[(.*)\]$", datasets_dtype)
|
| 151 |
+
if timestamp_matches:
|
| 152 |
+
timestamp_internals = timestamp_matches.group(1)
|
| 153 |
+
internals_matches = re.search(r"^(s|ms|us|ns),\s*tz=([a-zA-Z0-9/_+\-:]*)$", timestamp_internals)
|
| 154 |
+
if timestamp_internals in ["s", "ms", "us", "ns"]:
|
| 155 |
+
return pa.timestamp(timestamp_internals)
|
| 156 |
+
elif internals_matches:
|
| 157 |
+
return pa.timestamp(internals_matches.group(1), internals_matches.group(2))
|
| 158 |
+
else:
|
| 159 |
+
raise ValueError(
|
| 160 |
+
_dtype_error_msg(
|
| 161 |
+
datasets_dtype,
|
| 162 |
+
"timestamp",
|
| 163 |
+
examples=["timestamp[us]", "timestamp[us, tz=America/New_York"],
|
| 164 |
+
urls=["https://arrow.apache.org/docs/python/generated/pyarrow.timestamp.html"],
|
| 165 |
+
)
|
| 166 |
+
)
|
| 167 |
+
|
| 168 |
+
duration_matches = re.search(r"^duration\[(.*)\]$", datasets_dtype)
|
| 169 |
+
if duration_matches:
|
| 170 |
+
duration_internals = duration_matches.group(1)
|
| 171 |
+
if duration_internals in ["s", "ms", "us", "ns"]:
|
| 172 |
+
return pa.duration(duration_internals)
|
| 173 |
+
else:
|
| 174 |
+
raise ValueError(
|
| 175 |
+
_dtype_error_msg(
|
| 176 |
+
datasets_dtype,
|
| 177 |
+
"duration",
|
| 178 |
+
examples=["duration[s]", "duration[us]"],
|
| 179 |
+
urls=["https://arrow.apache.org/docs/python/generated/pyarrow.duration.html"],
|
| 180 |
+
)
|
| 181 |
+
)
|
| 182 |
+
|
| 183 |
+
time_matches = re.search(r"^time(.*)\[(.*)\]$", datasets_dtype)
|
| 184 |
+
if time_matches:
|
| 185 |
+
time_internals_bits = time_matches.group(1)
|
| 186 |
+
if time_internals_bits == "32":
|
| 187 |
+
time_internals_unit = time_matches.group(2)
|
| 188 |
+
if time_internals_unit in ["s", "ms"]:
|
| 189 |
+
return pa.time32(time_internals_unit)
|
| 190 |
+
else:
|
| 191 |
+
raise ValueError(
|
| 192 |
+
f"{time_internals_unit} is not a valid unit for the pyarrow time32 type. Supported units: s (second) and ms (millisecond)."
|
| 193 |
+
)
|
| 194 |
+
elif time_internals_bits == "64":
|
| 195 |
+
time_internals_unit = time_matches.group(2)
|
| 196 |
+
if time_internals_unit in ["us", "ns"]:
|
| 197 |
+
return pa.time64(time_internals_unit)
|
| 198 |
+
else:
|
| 199 |
+
raise ValueError(
|
| 200 |
+
f"{time_internals_unit} is not a valid unit for the pyarrow time64 type. Supported units: us (microsecond) and ns (nanosecond)."
|
| 201 |
+
)
|
| 202 |
+
else:
|
| 203 |
+
raise ValueError(
|
| 204 |
+
_dtype_error_msg(
|
| 205 |
+
datasets_dtype,
|
| 206 |
+
"time",
|
| 207 |
+
examples=["time32[s]", "time64[us]"],
|
| 208 |
+
urls=[
|
| 209 |
+
"https://arrow.apache.org/docs/python/generated/pyarrow.time32.html",
|
| 210 |
+
"https://arrow.apache.org/docs/python/generated/pyarrow.time64.html",
|
| 211 |
+
],
|
| 212 |
+
)
|
| 213 |
+
)
|
| 214 |
+
|
| 215 |
+
decimal_matches = re.search(r"^decimal(.*)\((.*)\)$", datasets_dtype)
|
| 216 |
+
if decimal_matches:
|
| 217 |
+
decimal_internals_bits = decimal_matches.group(1)
|
| 218 |
+
if decimal_internals_bits == "128":
|
| 219 |
+
decimal_internals_precision_and_scale = re.search(r"^(\d+),\s*(-?\d+)$", decimal_matches.group(2))
|
| 220 |
+
if decimal_internals_precision_and_scale:
|
| 221 |
+
precision = decimal_internals_precision_and_scale.group(1)
|
| 222 |
+
scale = decimal_internals_precision_and_scale.group(2)
|
| 223 |
+
return pa.decimal128(int(precision), int(scale))
|
| 224 |
+
else:
|
| 225 |
+
raise ValueError(
|
| 226 |
+
_dtype_error_msg(
|
| 227 |
+
datasets_dtype,
|
| 228 |
+
"decimal128",
|
| 229 |
+
examples=["decimal128(10, 2)", "decimal128(4, -2)"],
|
| 230 |
+
urls=["https://arrow.apache.org/docs/python/generated/pyarrow.decimal128.html"],
|
| 231 |
+
)
|
| 232 |
+
)
|
| 233 |
+
elif decimal_internals_bits == "256":
|
| 234 |
+
decimal_internals_precision_and_scale = re.search(r"^(\d+),\s*(-?\d+)$", decimal_matches.group(2))
|
| 235 |
+
if decimal_internals_precision_and_scale:
|
| 236 |
+
precision = decimal_internals_precision_and_scale.group(1)
|
| 237 |
+
scale = decimal_internals_precision_and_scale.group(2)
|
| 238 |
+
return pa.decimal256(int(precision), int(scale))
|
| 239 |
+
else:
|
| 240 |
+
raise ValueError(
|
| 241 |
+
_dtype_error_msg(
|
| 242 |
+
datasets_dtype,
|
| 243 |
+
"decimal256",
|
| 244 |
+
examples=["decimal256(30, 2)", "decimal256(38, -4)"],
|
| 245 |
+
urls=["https://arrow.apache.org/docs/python/generated/pyarrow.decimal256.html"],
|
| 246 |
+
)
|
| 247 |
+
)
|
| 248 |
+
else:
|
| 249 |
+
raise ValueError(
|
| 250 |
+
_dtype_error_msg(
|
| 251 |
+
datasets_dtype,
|
| 252 |
+
"decimal",
|
| 253 |
+
examples=["decimal128(12, 3)", "decimal256(40, 6)"],
|
| 254 |
+
urls=[
|
| 255 |
+
"https://arrow.apache.org/docs/python/generated/pyarrow.decimal128.html",
|
| 256 |
+
"https://arrow.apache.org/docs/python/generated/pyarrow.decimal256.html",
|
| 257 |
+
],
|
| 258 |
+
)
|
| 259 |
+
)
|
| 260 |
+
|
| 261 |
+
raise ValueError(
|
| 262 |
+
f"Neither {datasets_dtype} nor {datasets_dtype + '_'} seems to be a pyarrow data type. "
|
| 263 |
+
f"Please make sure to use a correct data type, see: "
|
| 264 |
+
f"https://arrow.apache.org/docs/python/api/datatypes.html#factory-functions"
|
| 265 |
+
)
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
def _cast_to_python_objects(obj: Any, only_1d_for_numpy: bool, optimize_list_casting: bool) -> tuple[Any, bool]:
|
| 269 |
+
"""
|
| 270 |
+
Cast pytorch/tensorflow/pandas objects to python numpy array/lists.
|
| 271 |
+
It works recursively.
|
| 272 |
+
|
| 273 |
+
If `optimize_list_casting` is True, to avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be casted.
|
| 274 |
+
If the first element needs to be casted, then all the elements of the list will be casted, otherwise they'll stay the same.
|
| 275 |
+
This trick allows to cast objects that contain tokenizers outputs without iterating over every single token for example.
|
| 276 |
+
|
| 277 |
+
Args:
|
| 278 |
+
obj: the object (nested struct) to cast.
|
| 279 |
+
only_1d_for_numpy (bool): whether to keep the full multi-dim tensors as multi-dim numpy arrays, or convert them to
|
| 280 |
+
nested lists of 1-dimensional numpy arrays. This can be useful to keep only 1-d arrays to instantiate Arrow arrays.
|
| 281 |
+
Indeed Arrow only support converting 1-dimensional array values.
|
| 282 |
+
optimize_list_casting (bool): whether to optimize list casting by checking the first non-null element to see if it needs to be casted
|
| 283 |
+
and if it doesn't, not checking the rest of the list elements.
|
| 284 |
+
|
| 285 |
+
Returns:
|
| 286 |
+
casted_obj: the casted object
|
| 287 |
+
has_changed (bool): True if the object has been changed, False if it is identical
|
| 288 |
+
"""
|
| 289 |
+
|
| 290 |
+
if config.TF_AVAILABLE and "tensorflow" in sys.modules:
|
| 291 |
+
import tensorflow as tf
|
| 292 |
+
|
| 293 |
+
if config.TORCH_AVAILABLE and "torch" in sys.modules:
|
| 294 |
+
import torch
|
| 295 |
+
|
| 296 |
+
if config.JAX_AVAILABLE and "jax" in sys.modules:
|
| 297 |
+
import jax.numpy as jnp
|
| 298 |
+
|
| 299 |
+
if config.PIL_AVAILABLE and "PIL" in sys.modules:
|
| 300 |
+
import PIL.Image
|
| 301 |
+
|
| 302 |
+
if config.PDFPLUMBER_AVAILABLE and "pdfplumber" in sys.modules:
|
| 303 |
+
import pdfplumber
|
| 304 |
+
|
| 305 |
+
if isinstance(obj, np.ndarray):
|
| 306 |
+
if obj.ndim == 0:
|
| 307 |
+
return obj[()], True
|
| 308 |
+
elif not only_1d_for_numpy or obj.ndim == 1:
|
| 309 |
+
return obj, False
|
| 310 |
+
else:
|
| 311 |
+
return (
|
| 312 |
+
[
|
| 313 |
+
_cast_to_python_objects(
|
| 314 |
+
x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
|
| 315 |
+
)[0]
|
| 316 |
+
for x in obj
|
| 317 |
+
],
|
| 318 |
+
True,
|
| 319 |
+
)
|
| 320 |
+
elif config.TORCH_AVAILABLE and "torch" in sys.modules and isinstance(obj, torch.Tensor):
|
| 321 |
+
if obj.dtype == torch.bfloat16:
|
| 322 |
+
return _cast_to_python_objects(
|
| 323 |
+
obj.detach().to(torch.float).cpu().numpy(),
|
| 324 |
+
only_1d_for_numpy=only_1d_for_numpy,
|
| 325 |
+
optimize_list_casting=optimize_list_casting,
|
| 326 |
+
)[0], True
|
| 327 |
+
if obj.ndim == 0:
|
| 328 |
+
return obj.detach().cpu().numpy()[()], True
|
| 329 |
+
elif not only_1d_for_numpy or obj.ndim == 1:
|
| 330 |
+
return obj.detach().cpu().numpy(), True
|
| 331 |
+
else:
|
| 332 |
+
return (
|
| 333 |
+
[
|
| 334 |
+
_cast_to_python_objects(
|
| 335 |
+
x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
|
| 336 |
+
)[0]
|
| 337 |
+
for x in obj.detach().cpu().numpy()
|
| 338 |
+
],
|
| 339 |
+
True,
|
| 340 |
+
)
|
| 341 |
+
elif config.TF_AVAILABLE and "tensorflow" in sys.modules and isinstance(obj, tf.Tensor):
|
| 342 |
+
if obj.ndim == 0:
|
| 343 |
+
return obj.numpy()[()], True
|
| 344 |
+
elif not only_1d_for_numpy or obj.ndim == 1:
|
| 345 |
+
return obj.numpy(), True
|
| 346 |
+
else:
|
| 347 |
+
return (
|
| 348 |
+
[
|
| 349 |
+
_cast_to_python_objects(
|
| 350 |
+
x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
|
| 351 |
+
)[0]
|
| 352 |
+
for x in obj.numpy()
|
| 353 |
+
],
|
| 354 |
+
True,
|
| 355 |
+
)
|
| 356 |
+
elif config.JAX_AVAILABLE and "jax" in sys.modules and isinstance(obj, jnp.ndarray):
|
| 357 |
+
if obj.ndim == 0:
|
| 358 |
+
return np.asarray(obj)[()], True
|
| 359 |
+
elif not only_1d_for_numpy or obj.ndim == 1:
|
| 360 |
+
return np.asarray(obj), True
|
| 361 |
+
else:
|
| 362 |
+
return (
|
| 363 |
+
[
|
| 364 |
+
_cast_to_python_objects(
|
| 365 |
+
x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
|
| 366 |
+
)[0]
|
| 367 |
+
for x in np.asarray(obj)
|
| 368 |
+
],
|
| 369 |
+
True,
|
| 370 |
+
)
|
| 371 |
+
elif config.PIL_AVAILABLE and "PIL" in sys.modules and isinstance(obj, PIL.Image.Image):
|
| 372 |
+
return encode_pil_image(obj), True
|
| 373 |
+
elif config.PDFPLUMBER_AVAILABLE and "pdfplumber" in sys.modules and isinstance(obj, pdfplumber.pdf.PDF):
|
| 374 |
+
return encode_pdfplumber_pdf(obj), True
|
| 375 |
+
elif isinstance(obj, pd.Series):
|
| 376 |
+
return (
|
| 377 |
+
_cast_to_python_objects(
|
| 378 |
+
obj.tolist(), only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
|
| 379 |
+
)[0],
|
| 380 |
+
True,
|
| 381 |
+
)
|
| 382 |
+
elif isinstance(obj, pd.DataFrame):
|
| 383 |
+
return (
|
| 384 |
+
{
|
| 385 |
+
key: _cast_to_python_objects(
|
| 386 |
+
value, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
|
| 387 |
+
)[0]
|
| 388 |
+
for key, value in obj.to_dict("series").items()
|
| 389 |
+
},
|
| 390 |
+
True,
|
| 391 |
+
)
|
| 392 |
+
elif isinstance(obj, pd.Timestamp):
|
| 393 |
+
return obj.to_pydatetime(), True
|
| 394 |
+
elif isinstance(obj, pd.Timedelta):
|
| 395 |
+
return obj.to_pytimedelta(), True
|
| 396 |
+
elif isinstance(obj, Mapping):
|
| 397 |
+
has_changed = not isinstance(obj, dict)
|
| 398 |
+
output = {}
|
| 399 |
+
for k, v in obj.items():
|
| 400 |
+
casted_v, has_changed_v = _cast_to_python_objects(
|
| 401 |
+
v, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
|
| 402 |
+
)
|
| 403 |
+
has_changed |= has_changed_v
|
| 404 |
+
output[k] = casted_v
|
| 405 |
+
return output if has_changed else obj, has_changed
|
| 406 |
+
elif hasattr(obj, "__array__"):
|
| 407 |
+
if np.isscalar(obj):
|
| 408 |
+
return obj, False
|
| 409 |
+
else:
|
| 410 |
+
return (
|
| 411 |
+
_cast_to_python_objects(
|
| 412 |
+
obj.__array__(), only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
|
| 413 |
+
)[0],
|
| 414 |
+
True,
|
| 415 |
+
)
|
| 416 |
+
elif isinstance(obj, (list, tuple)):
|
| 417 |
+
if len(obj) > 0:
|
| 418 |
+
for first_elmt in obj:
|
| 419 |
+
if _check_non_null_non_empty_recursive(first_elmt):
|
| 420 |
+
break
|
| 421 |
+
casted_first_elmt, has_changed_first_elmt = _cast_to_python_objects(
|
| 422 |
+
first_elmt, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
|
| 423 |
+
)
|
| 424 |
+
if has_changed_first_elmt or not optimize_list_casting:
|
| 425 |
+
return (
|
| 426 |
+
[
|
| 427 |
+
_cast_to_python_objects(
|
| 428 |
+
elmt, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
|
| 429 |
+
)[0]
|
| 430 |
+
for elmt in obj
|
| 431 |
+
],
|
| 432 |
+
True,
|
| 433 |
+
)
|
| 434 |
+
else:
|
| 435 |
+
if isinstance(obj, (list, tuple)):
|
| 436 |
+
return obj, False
|
| 437 |
+
else:
|
| 438 |
+
return list(obj), True
|
| 439 |
+
else:
|
| 440 |
+
return obj, False
|
| 441 |
+
else:
|
| 442 |
+
return obj, False
|
| 443 |
+
|
| 444 |
+
|
| 445 |
+
def cast_to_python_objects(obj: Any, only_1d_for_numpy=False, optimize_list_casting=True) -> Any:
|
| 446 |
+
"""
|
| 447 |
+
Cast numpy/pytorch/tensorflow/pandas objects to python lists.
|
| 448 |
+
It works recursively.
|
| 449 |
+
|
| 450 |
+
If `optimize_list_casting` is True, To avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be casted.
|
| 451 |
+
If the first element needs to be casted, then all the elements of the list will be casted, otherwise they'll stay the same.
|
| 452 |
+
This trick allows to cast objects that contain tokenizers outputs without iterating over every single token for example.
|
| 453 |
+
|
| 454 |
+
Args:
|
| 455 |
+
obj: the object (nested struct) to cast
|
| 456 |
+
only_1d_for_numpy (bool, default ``False``): whether to keep the full multi-dim tensors as multi-dim numpy arrays, or convert them to
|
| 457 |
+
nested lists of 1-dimensional numpy arrays. This can be useful to keep only 1-d arrays to instantiate Arrow arrays.
|
| 458 |
+
Indeed Arrow only support converting 1-dimensional array values.
|
| 459 |
+
optimize_list_casting (bool, default ``True``): whether to optimize list casting by checking the first non-null element to see if it needs to be casted
|
| 460 |
+
and if it doesn't, not checking the rest of the list elements.
|
| 461 |
+
|
| 462 |
+
Returns:
|
| 463 |
+
casted_obj: the casted object
|
| 464 |
+
"""
|
| 465 |
+
return _cast_to_python_objects(
|
| 466 |
+
obj, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
|
| 467 |
+
)[0]
|
| 468 |
+
|
| 469 |
+
|
| 470 |
+
@dataclass
|
| 471 |
+
class Value:
|
| 472 |
+
"""
|
| 473 |
+
Scalar feature value of a particular data type.
|
| 474 |
+
|
| 475 |
+
The possible dtypes of `Value` are as follows:
|
| 476 |
+
- `null`
|
| 477 |
+
- `bool`
|
| 478 |
+
- `int8`
|
| 479 |
+
- `int16`
|
| 480 |
+
- `int32`
|
| 481 |
+
- `int64`
|
| 482 |
+
- `uint8`
|
| 483 |
+
- `uint16`
|
| 484 |
+
- `uint32`
|
| 485 |
+
- `uint64`
|
| 486 |
+
- `float16`
|
| 487 |
+
- `float32` (alias float)
|
| 488 |
+
- `float64` (alias double)
|
| 489 |
+
- `time32[(s|ms)]`
|
| 490 |
+
- `time64[(us|ns)]`
|
| 491 |
+
- `timestamp[(s|ms|us|ns)]`
|
| 492 |
+
- `timestamp[(s|ms|us|ns), tz=(tzstring)]`
|
| 493 |
+
- `date32`
|
| 494 |
+
- `date64`
|
| 495 |
+
- `duration[(s|ms|us|ns)]`
|
| 496 |
+
- `decimal128(precision, scale)`
|
| 497 |
+
- `decimal256(precision, scale)`
|
| 498 |
+
- `binary`
|
| 499 |
+
- `large_binary`
|
| 500 |
+
- `string`
|
| 501 |
+
- `large_string`
|
| 502 |
+
|
| 503 |
+
Args:
|
| 504 |
+
dtype (`str`):
|
| 505 |
+
Name of the data type.
|
| 506 |
+
|
| 507 |
+
Example:
|
| 508 |
+
|
| 509 |
+
```py
|
| 510 |
+
>>> from datasets import Features
|
| 511 |
+
>>> features = Features({'stars': Value(dtype='int32')})
|
| 512 |
+
>>> features
|
| 513 |
+
{'stars': Value(dtype='int32', id=None)}
|
| 514 |
+
```
|
| 515 |
+
"""
|
| 516 |
+
|
| 517 |
+
dtype: str
|
| 518 |
+
id: Optional[str] = None
|
| 519 |
+
# Automatically constructed
|
| 520 |
+
pa_type: ClassVar[Any] = None
|
| 521 |
+
_type: str = field(default="Value", init=False, repr=False)
|
| 522 |
+
|
| 523 |
+
def __post_init__(self):
|
| 524 |
+
if self.dtype == "double": # fix inferred type
|
| 525 |
+
self.dtype = "float64"
|
| 526 |
+
if self.dtype == "float": # fix inferred type
|
| 527 |
+
self.dtype = "float32"
|
| 528 |
+
self.pa_type = string_to_arrow(self.dtype)
|
| 529 |
+
|
| 530 |
+
def __call__(self):
|
| 531 |
+
return self.pa_type
|
| 532 |
+
|
| 533 |
+
def encode_example(self, value):
|
| 534 |
+
if pa.types.is_boolean(self.pa_type):
|
| 535 |
+
return bool(value)
|
| 536 |
+
elif pa.types.is_integer(self.pa_type):
|
| 537 |
+
return int(value)
|
| 538 |
+
elif pa.types.is_floating(self.pa_type):
|
| 539 |
+
return float(value)
|
| 540 |
+
elif pa.types.is_string(self.pa_type):
|
| 541 |
+
return str(value)
|
| 542 |
+
else:
|
| 543 |
+
return value
|
| 544 |
+
|
| 545 |
+
|
| 546 |
+
class _ArrayXD:
|
| 547 |
+
def __post_init__(self):
|
| 548 |
+
self.shape = tuple(self.shape)
|
| 549 |
+
|
| 550 |
+
def __call__(self):
|
| 551 |
+
pa_type = globals()[self.__class__.__name__ + "ExtensionType"](self.shape, self.dtype)
|
| 552 |
+
return pa_type
|
| 553 |
+
|
| 554 |
+
def encode_example(self, value):
|
| 555 |
+
return value
|
| 556 |
+
|
| 557 |
+
|
| 558 |
+
@dataclass
|
| 559 |
+
class Array2D(_ArrayXD):
|
| 560 |
+
"""Create a two-dimensional array.
|
| 561 |
+
|
| 562 |
+
Args:
|
| 563 |
+
shape (`tuple`):
|
| 564 |
+
Size of each dimension.
|
| 565 |
+
dtype (`str`):
|
| 566 |
+
Name of the data type.
|
| 567 |
+
|
| 568 |
+
Example:
|
| 569 |
+
|
| 570 |
+
```py
|
| 571 |
+
>>> from datasets import Features
|
| 572 |
+
>>> features = Features({'x': Array2D(shape=(1, 3), dtype='int32')})
|
| 573 |
+
```
|
| 574 |
+
"""
|
| 575 |
+
|
| 576 |
+
shape: tuple
|
| 577 |
+
dtype: str
|
| 578 |
+
id: Optional[str] = None
|
| 579 |
+
# Automatically constructed
|
| 580 |
+
_type: str = field(default="Array2D", init=False, repr=False)
|
| 581 |
+
|
| 582 |
+
|
| 583 |
+
@dataclass
|
| 584 |
+
class Array3D(_ArrayXD):
|
| 585 |
+
"""Create a three-dimensional array.
|
| 586 |
+
|
| 587 |
+
Args:
|
| 588 |
+
shape (`tuple`):
|
| 589 |
+
Size of each dimension.
|
| 590 |
+
dtype (`str`):
|
| 591 |
+
Name of the data type.
|
| 592 |
+
|
| 593 |
+
Example:
|
| 594 |
+
|
| 595 |
+
```py
|
| 596 |
+
>>> from datasets import Features
|
| 597 |
+
>>> features = Features({'x': Array3D(shape=(1, 2, 3), dtype='int32')})
|
| 598 |
+
```
|
| 599 |
+
"""
|
| 600 |
+
|
| 601 |
+
shape: tuple
|
| 602 |
+
dtype: str
|
| 603 |
+
id: Optional[str] = None
|
| 604 |
+
# Automatically constructed
|
| 605 |
+
_type: str = field(default="Array3D", init=False, repr=False)
|
| 606 |
+
|
| 607 |
+
|
| 608 |
+
@dataclass
|
| 609 |
+
class Array4D(_ArrayXD):
|
| 610 |
+
"""Create a four-dimensional array.
|
| 611 |
+
|
| 612 |
+
Args:
|
| 613 |
+
shape (`tuple`):
|
| 614 |
+
Size of each dimension.
|
| 615 |
+
dtype (`str`):
|
| 616 |
+
Name of the data type.
|
| 617 |
+
|
| 618 |
+
Example:
|
| 619 |
+
|
| 620 |
+
```py
|
| 621 |
+
>>> from datasets import Features
|
| 622 |
+
>>> features = Features({'x': Array4D(shape=(1, 2, 2, 3), dtype='int32')})
|
| 623 |
+
```
|
| 624 |
+
"""
|
| 625 |
+
|
| 626 |
+
shape: tuple
|
| 627 |
+
dtype: str
|
| 628 |
+
id: Optional[str] = None
|
| 629 |
+
# Automatically constructed
|
| 630 |
+
_type: str = field(default="Array4D", init=False, repr=False)
|
| 631 |
+
|
| 632 |
+
|
| 633 |
+
@dataclass
|
| 634 |
+
class Array5D(_ArrayXD):
|
| 635 |
+
"""Create a five-dimensional array.
|
| 636 |
+
|
| 637 |
+
Args:
|
| 638 |
+
shape (`tuple`):
|
| 639 |
+
Size of each dimension.
|
| 640 |
+
dtype (`str`):
|
| 641 |
+
Name of the data type.
|
| 642 |
+
|
| 643 |
+
Example:
|
| 644 |
+
|
| 645 |
+
```py
|
| 646 |
+
>>> from datasets import Features
|
| 647 |
+
>>> features = Features({'x': Array5D(shape=(1, 2, 2, 3, 3), dtype='int32')})
|
| 648 |
+
```
|
| 649 |
+
"""
|
| 650 |
+
|
| 651 |
+
shape: tuple
|
| 652 |
+
dtype: str
|
| 653 |
+
id: Optional[str] = None
|
| 654 |
+
# Automatically constructed
|
| 655 |
+
_type: str = field(default="Array5D", init=False, repr=False)
|
| 656 |
+
|
| 657 |
+
|
| 658 |
+
class _ArrayXDExtensionType(pa.ExtensionType):
|
| 659 |
+
ndims: Optional[int] = None
|
| 660 |
+
|
| 661 |
+
def __init__(self, shape: tuple, dtype: str):
|
| 662 |
+
if self.ndims is None or self.ndims <= 1:
|
| 663 |
+
raise ValueError("You must instantiate an array type with a value for dim that is > 1")
|
| 664 |
+
if len(shape) != self.ndims:
|
| 665 |
+
raise ValueError(f"shape={shape} and ndims={self.ndims} don't match")
|
| 666 |
+
for dim in range(1, self.ndims):
|
| 667 |
+
if shape[dim] is None:
|
| 668 |
+
raise ValueError(f"Support only dynamic size on first dimension. Got: {shape}")
|
| 669 |
+
self.shape = tuple(shape)
|
| 670 |
+
self.value_type = dtype
|
| 671 |
+
self.storage_dtype = self._generate_dtype(self.value_type)
|
| 672 |
+
pa.ExtensionType.__init__(self, self.storage_dtype, f"{self.__class__.__module__}.{self.__class__.__name__}")
|
| 673 |
+
|
| 674 |
+
def __arrow_ext_serialize__(self):
|
| 675 |
+
return json.dumps((self.shape, self.value_type)).encode()
|
| 676 |
+
|
| 677 |
+
@classmethod
|
| 678 |
+
def __arrow_ext_deserialize__(cls, storage_type, serialized):
|
| 679 |
+
args = json.loads(serialized)
|
| 680 |
+
return cls(*args)
|
| 681 |
+
|
| 682 |
+
# This was added to pa.ExtensionType in pyarrow >= 13.0.0
|
| 683 |
+
def __reduce__(self):
|
| 684 |
+
return self.__arrow_ext_deserialize__, (self.storage_type, self.__arrow_ext_serialize__())
|
| 685 |
+
|
| 686 |
+
def __hash__(self):
|
| 687 |
+
return hash((self.__class__, self.shape, self.value_type))
|
| 688 |
+
|
| 689 |
+
def __arrow_ext_class__(self):
|
| 690 |
+
return ArrayExtensionArray
|
| 691 |
+
|
| 692 |
+
def _generate_dtype(self, dtype):
|
| 693 |
+
dtype = string_to_arrow(dtype)
|
| 694 |
+
for d in reversed(self.shape):
|
| 695 |
+
dtype = pa.list_(dtype)
|
| 696 |
+
# Don't specify the size of the list, since fixed length list arrays have issues
|
| 697 |
+
# being validated after slicing in pyarrow 0.17.1
|
| 698 |
+
return dtype
|
| 699 |
+
|
| 700 |
+
def to_pandas_dtype(self):
|
| 701 |
+
return PandasArrayExtensionDtype(self.value_type)
|
| 702 |
+
|
| 703 |
+
|
| 704 |
+
class Array2DExtensionType(_ArrayXDExtensionType):
|
| 705 |
+
ndims = 2
|
| 706 |
+
|
| 707 |
+
|
| 708 |
+
class Array3DExtensionType(_ArrayXDExtensionType):
|
| 709 |
+
ndims = 3
|
| 710 |
+
|
| 711 |
+
|
| 712 |
+
class Array4DExtensionType(_ArrayXDExtensionType):
|
| 713 |
+
ndims = 4
|
| 714 |
+
|
| 715 |
+
|
| 716 |
+
class Array5DExtensionType(_ArrayXDExtensionType):
|
| 717 |
+
ndims = 5
|
| 718 |
+
|
| 719 |
+
|
| 720 |
+
# Register the extension types for deserialization
|
| 721 |
+
pa.register_extension_type(Array2DExtensionType((1, 2), "int64"))
|
| 722 |
+
pa.register_extension_type(Array3DExtensionType((1, 2, 3), "int64"))
|
| 723 |
+
pa.register_extension_type(Array4DExtensionType((1, 2, 3, 4), "int64"))
|
| 724 |
+
pa.register_extension_type(Array5DExtensionType((1, 2, 3, 4, 5), "int64"))
|
| 725 |
+
|
| 726 |
+
|
| 727 |
+
def _is_zero_copy_only(pa_type: pa.DataType, unnest: bool = False) -> bool:
|
| 728 |
+
"""
|
| 729 |
+
When converting a pyarrow array to a numpy array, we must know whether this could be done in zero-copy or not.
|
| 730 |
+
This function returns the value of the ``zero_copy_only`` parameter to pass to ``.to_numpy()``, given the type of the pyarrow array.
|
| 731 |
+
|
| 732 |
+
# zero copy is available for all primitive types except booleans and temporal types (date, time, timestamp or duration)
|
| 733 |
+
# primitive types are types for which the physical representation in arrow and in numpy
|
| 734 |
+
# https://github.com/wesm/arrow/blob/c07b9b48cf3e0bbbab493992a492ae47e5b04cad/python/pyarrow/types.pxi#L821
|
| 735 |
+
# see https://arrow.apache.org/docs/python/generated/pyarrow.Array.html#pyarrow.Array.to_numpy
|
| 736 |
+
# and https://issues.apache.org/jira/browse/ARROW-2871?jql=text%20~%20%22boolean%20to_numpy%22
|
| 737 |
+
"""
|
| 738 |
+
|
| 739 |
+
def _unnest_pa_type(pa_type: pa.DataType) -> pa.DataType:
|
| 740 |
+
if pa.types.is_list(pa_type):
|
| 741 |
+
return _unnest_pa_type(pa_type.value_type)
|
| 742 |
+
return pa_type
|
| 743 |
+
|
| 744 |
+
if unnest:
|
| 745 |
+
pa_type = _unnest_pa_type(pa_type)
|
| 746 |
+
return pa.types.is_primitive(pa_type) and not (pa.types.is_boolean(pa_type) or pa.types.is_temporal(pa_type))
|
| 747 |
+
|
| 748 |
+
|
| 749 |
+
class ArrayExtensionArray(pa.ExtensionArray):
|
| 750 |
+
def __array__(self):
|
| 751 |
+
zero_copy_only = _is_zero_copy_only(self.storage.type, unnest=True)
|
| 752 |
+
return self.to_numpy(zero_copy_only=zero_copy_only)
|
| 753 |
+
|
| 754 |
+
def __getitem__(self, i):
|
| 755 |
+
return self.storage[i]
|
| 756 |
+
|
| 757 |
+
def to_numpy(self, zero_copy_only=True):
|
| 758 |
+
storage: pa.ListArray = self.storage
|
| 759 |
+
null_mask = storage.is_null().to_numpy(zero_copy_only=False)
|
| 760 |
+
|
| 761 |
+
if self.type.shape[0] is not None:
|
| 762 |
+
size = 1
|
| 763 |
+
null_indices = np.arange(len(storage))[null_mask] - np.arange(np.sum(null_mask))
|
| 764 |
+
|
| 765 |
+
for i in range(self.type.ndims):
|
| 766 |
+
size *= self.type.shape[i]
|
| 767 |
+
storage = storage.flatten()
|
| 768 |
+
numpy_arr = storage.to_numpy(zero_copy_only=zero_copy_only)
|
| 769 |
+
numpy_arr = numpy_arr.reshape(len(self) - len(null_indices), *self.type.shape)
|
| 770 |
+
|
| 771 |
+
if len(null_indices):
|
| 772 |
+
numpy_arr = np.insert(numpy_arr.astype(np.float64), null_indices, np.nan, axis=0)
|
| 773 |
+
|
| 774 |
+
else:
|
| 775 |
+
shape = self.type.shape
|
| 776 |
+
ndims = self.type.ndims
|
| 777 |
+
arrays = []
|
| 778 |
+
first_dim_offsets = np.array([off.as_py() for off in storage.offsets])
|
| 779 |
+
for i, is_null in enumerate(null_mask):
|
| 780 |
+
if is_null:
|
| 781 |
+
arrays.append(np.nan)
|
| 782 |
+
else:
|
| 783 |
+
storage_el = storage[i : i + 1]
|
| 784 |
+
first_dim = first_dim_offsets[i + 1] - first_dim_offsets[i]
|
| 785 |
+
# flatten storage
|
| 786 |
+
for _ in range(ndims):
|
| 787 |
+
storage_el = storage_el.flatten()
|
| 788 |
+
|
| 789 |
+
numpy_arr = storage_el.to_numpy(zero_copy_only=zero_copy_only)
|
| 790 |
+
arrays.append(numpy_arr.reshape(first_dim, *shape[1:]))
|
| 791 |
+
|
| 792 |
+
if len(np.unique(np.diff(first_dim_offsets))) > 1:
|
| 793 |
+
# ragged
|
| 794 |
+
numpy_arr = np.empty(len(arrays), dtype=object)
|
| 795 |
+
numpy_arr[:] = arrays
|
| 796 |
+
else:
|
| 797 |
+
numpy_arr = np.array(arrays)
|
| 798 |
+
|
| 799 |
+
return numpy_arr
|
| 800 |
+
|
| 801 |
+
def to_pylist(self, maps_as_pydicts: Optional[Literal["lossy", "strict"]] = None):
|
| 802 |
+
zero_copy_only = _is_zero_copy_only(self.storage.type, unnest=True)
|
| 803 |
+
numpy_arr = self.to_numpy(zero_copy_only=zero_copy_only)
|
| 804 |
+
if self.type.shape[0] is None and numpy_arr.dtype == object:
|
| 805 |
+
return [arr.tolist() for arr in numpy_arr.tolist()]
|
| 806 |
+
else:
|
| 807 |
+
return numpy_arr.tolist()
|
| 808 |
+
|
| 809 |
+
|
| 810 |
+
class PandasArrayExtensionDtype(PandasExtensionDtype):
|
| 811 |
+
_metadata = "value_type"
|
| 812 |
+
|
| 813 |
+
def __init__(self, value_type: Union["PandasArrayExtensionDtype", np.dtype]):
|
| 814 |
+
self._value_type = value_type
|
| 815 |
+
|
| 816 |
+
def __from_arrow__(self, array: Union[pa.Array, pa.ChunkedArray]):
|
| 817 |
+
if isinstance(array, pa.ChunkedArray):
|
| 818 |
+
array = array.type.wrap_array(pa.concat_arrays([chunk.storage for chunk in array.chunks]))
|
| 819 |
+
zero_copy_only = _is_zero_copy_only(array.storage.type, unnest=True)
|
| 820 |
+
numpy_arr = array.to_numpy(zero_copy_only=zero_copy_only)
|
| 821 |
+
return PandasArrayExtensionArray(numpy_arr)
|
| 822 |
+
|
| 823 |
+
@classmethod
|
| 824 |
+
def construct_array_type(cls):
|
| 825 |
+
return PandasArrayExtensionArray
|
| 826 |
+
|
| 827 |
+
@property
|
| 828 |
+
def type(self) -> type:
|
| 829 |
+
return np.ndarray
|
| 830 |
+
|
| 831 |
+
@property
|
| 832 |
+
def kind(self) -> str:
|
| 833 |
+
return "O"
|
| 834 |
+
|
| 835 |
+
@property
|
| 836 |
+
def name(self) -> str:
|
| 837 |
+
return f"array[{self.value_type}]"
|
| 838 |
+
|
| 839 |
+
@property
|
| 840 |
+
def value_type(self) -> np.dtype:
|
| 841 |
+
return self._value_type
|
| 842 |
+
|
| 843 |
+
|
| 844 |
+
class PandasArrayExtensionArray(PandasExtensionArray):
|
| 845 |
+
def __init__(self, data: np.ndarray, copy: bool = False):
|
| 846 |
+
self._data = data if not copy else np.array(data)
|
| 847 |
+
self._dtype = PandasArrayExtensionDtype(data.dtype)
|
| 848 |
+
|
| 849 |
+
def __array__(self, dtype=None):
|
| 850 |
+
"""
|
| 851 |
+
Convert to NumPy Array.
|
| 852 |
+
Note that Pandas expects a 1D array when dtype is set to object.
|
| 853 |
+
But for other dtypes, the returned shape is the same as the one of ``data``.
|
| 854 |
+
|
| 855 |
+
More info about pandas 1D requirement for PandasExtensionArray here:
|
| 856 |
+
https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.api.extensions.ExtensionArray.html#pandas.api.extensions.ExtensionArray
|
| 857 |
+
|
| 858 |
+
"""
|
| 859 |
+
if dtype == np.dtype(object):
|
| 860 |
+
out = np.empty(len(self._data), dtype=object)
|
| 861 |
+
for i in range(len(self._data)):
|
| 862 |
+
out[i] = self._data[i]
|
| 863 |
+
return out
|
| 864 |
+
if dtype is None:
|
| 865 |
+
return self._data
|
| 866 |
+
else:
|
| 867 |
+
return self._data.astype(dtype)
|
| 868 |
+
|
| 869 |
+
def copy(self, deep: bool = False) -> "PandasArrayExtensionArray":
|
| 870 |
+
return PandasArrayExtensionArray(self._data, copy=True)
|
| 871 |
+
|
| 872 |
+
@classmethod
|
| 873 |
+
def _from_sequence(
|
| 874 |
+
cls, scalars, dtype: Optional[PandasArrayExtensionDtype] = None, copy: bool = False
|
| 875 |
+
) -> "PandasArrayExtensionArray":
|
| 876 |
+
if len(scalars) > 1 and all(
|
| 877 |
+
isinstance(x, np.ndarray) and x.shape == scalars[0].shape and x.dtype == scalars[0].dtype for x in scalars
|
| 878 |
+
):
|
| 879 |
+
data = np.array(scalars, dtype=dtype if dtype is None else dtype.value_type, copy=copy)
|
| 880 |
+
else:
|
| 881 |
+
data = np.empty(len(scalars), dtype=object)
|
| 882 |
+
data[:] = scalars
|
| 883 |
+
return cls(data, copy=copy)
|
| 884 |
+
|
| 885 |
+
@classmethod
|
| 886 |
+
def _concat_same_type(cls, to_concat: Sequence_["PandasArrayExtensionArray"]) -> "PandasArrayExtensionArray":
|
| 887 |
+
if len(to_concat) > 1 and all(
|
| 888 |
+
va._data.shape == to_concat[0]._data.shape and va._data.dtype == to_concat[0]._data.dtype
|
| 889 |
+
for va in to_concat
|
| 890 |
+
):
|
| 891 |
+
data = np.vstack([va._data for va in to_concat])
|
| 892 |
+
else:
|
| 893 |
+
data = np.empty(len(to_concat), dtype=object)
|
| 894 |
+
data[:] = [va._data for va in to_concat]
|
| 895 |
+
return cls(data, copy=False)
|
| 896 |
+
|
| 897 |
+
@property
|
| 898 |
+
def dtype(self) -> PandasArrayExtensionDtype:
|
| 899 |
+
return self._dtype
|
| 900 |
+
|
| 901 |
+
@property
|
| 902 |
+
def nbytes(self) -> int:
|
| 903 |
+
return self._data.nbytes
|
| 904 |
+
|
| 905 |
+
def isna(self) -> np.ndarray:
|
| 906 |
+
return np.array([pd.isna(arr).any() for arr in self._data])
|
| 907 |
+
|
| 908 |
+
def __setitem__(self, key: Union[int, slice, np.ndarray], value: Any) -> None:
|
| 909 |
+
raise NotImplementedError()
|
| 910 |
+
|
| 911 |
+
def __getitem__(self, item: Union[int, slice, np.ndarray]) -> Union[np.ndarray, "PandasArrayExtensionArray"]:
|
| 912 |
+
if isinstance(item, int):
|
| 913 |
+
return self._data[item]
|
| 914 |
+
return PandasArrayExtensionArray(self._data[item], copy=False)
|
| 915 |
+
|
| 916 |
+
def take(
|
| 917 |
+
self, indices: Sequence_[int], allow_fill: bool = False, fill_value: bool = None
|
| 918 |
+
) -> "PandasArrayExtensionArray":
|
| 919 |
+
indices: np.ndarray = np.asarray(indices, dtype=int)
|
| 920 |
+
if allow_fill:
|
| 921 |
+
fill_value = (
|
| 922 |
+
self.dtype.na_value if fill_value is None else np.asarray(fill_value, dtype=self.dtype.value_type)
|
| 923 |
+
)
|
| 924 |
+
mask = indices == -1
|
| 925 |
+
if (indices < -1).any():
|
| 926 |
+
raise ValueError("Invalid value in `indices`, must be all >= -1 for `allow_fill` is True")
|
| 927 |
+
elif len(self) > 0:
|
| 928 |
+
pass
|
| 929 |
+
elif not np.all(mask):
|
| 930 |
+
raise IndexError("Invalid take for empty PandasArrayExtensionArray, must be all -1.")
|
| 931 |
+
else:
|
| 932 |
+
data = np.array([fill_value] * len(indices), dtype=self.dtype.value_type)
|
| 933 |
+
return PandasArrayExtensionArray(data, copy=False)
|
| 934 |
+
took = self._data.take(indices, axis=0)
|
| 935 |
+
if allow_fill and mask.any():
|
| 936 |
+
took[mask] = [fill_value] * np.sum(mask)
|
| 937 |
+
return PandasArrayExtensionArray(took, copy=False)
|
| 938 |
+
|
| 939 |
+
def __len__(self) -> int:
|
| 940 |
+
return len(self._data)
|
| 941 |
+
|
| 942 |
+
def __eq__(self, other) -> np.ndarray:
|
| 943 |
+
if not isinstance(other, PandasArrayExtensionArray):
|
| 944 |
+
raise NotImplementedError(f"Invalid type to compare to: {type(other)}")
|
| 945 |
+
return (self._data == other._data).all()
|
| 946 |
+
|
| 947 |
+
|
| 948 |
+
def pandas_types_mapper(dtype):
|
| 949 |
+
if isinstance(dtype, _ArrayXDExtensionType):
|
| 950 |
+
return PandasArrayExtensionDtype(dtype.value_type)
|
| 951 |
+
|
| 952 |
+
|
| 953 |
+
@dataclass
|
| 954 |
+
class ClassLabel:
|
| 955 |
+
"""Feature type for integer class labels.
|
| 956 |
+
|
| 957 |
+
There are 3 ways to define a `ClassLabel`, which correspond to the 3 arguments:
|
| 958 |
+
|
| 959 |
+
* `num_classes`: Create 0 to (num_classes-1) labels.
|
| 960 |
+
* `names`: List of label strings.
|
| 961 |
+
* `names_file`: File containing the list of labels.
|
| 962 |
+
|
| 963 |
+
Under the hood the labels are stored as integers.
|
| 964 |
+
You can use negative integers to represent unknown/missing labels.
|
| 965 |
+
|
| 966 |
+
Args:
|
| 967 |
+
num_classes (`int`, *optional*):
|
| 968 |
+
Number of classes. All labels must be < `num_classes`.
|
| 969 |
+
names (`list` of `str`, *optional*):
|
| 970 |
+
String names for the integer classes.
|
| 971 |
+
The order in which the names are provided is kept.
|
| 972 |
+
names_file (`str`, *optional*):
|
| 973 |
+
Path to a file with names for the integer classes, one per line.
|
| 974 |
+
|
| 975 |
+
Example:
|
| 976 |
+
|
| 977 |
+
```py
|
| 978 |
+
>>> from datasets import Features, ClassLabel
|
| 979 |
+
>>> features = Features({'label': ClassLabel(num_classes=3, names=['bad', 'ok', 'good'])})
|
| 980 |
+
>>> features
|
| 981 |
+
{'label': ClassLabel(names=['bad', 'ok', 'good'], id=None)}
|
| 982 |
+
```
|
| 983 |
+
"""
|
| 984 |
+
|
| 985 |
+
num_classes: InitVar[Optional[int]] = None # Pseudo-field: ignored by asdict/fields when converting to/from dict
|
| 986 |
+
names: list[str] = None
|
| 987 |
+
names_file: InitVar[Optional[str]] = None # Pseudo-field: ignored by asdict/fields when converting to/from dict
|
| 988 |
+
id: Optional[str] = None
|
| 989 |
+
# Automatically constructed
|
| 990 |
+
dtype: ClassVar[str] = "int64"
|
| 991 |
+
pa_type: ClassVar[Any] = pa.int64()
|
| 992 |
+
_str2int: ClassVar[dict[str, int]] = None
|
| 993 |
+
_int2str: ClassVar[dict[int, int]] = None
|
| 994 |
+
_type: str = field(default="ClassLabel", init=False, repr=False)
|
| 995 |
+
|
| 996 |
+
def __post_init__(self, num_classes, names_file):
|
| 997 |
+
self.num_classes = num_classes
|
| 998 |
+
self.names_file = names_file
|
| 999 |
+
if self.names_file is not None and self.names is not None:
|
| 1000 |
+
raise ValueError("Please provide either names or names_file but not both.")
|
| 1001 |
+
# Set self.names
|
| 1002 |
+
if self.names is None:
|
| 1003 |
+
if self.names_file is not None:
|
| 1004 |
+
self.names = self._load_names_from_file(self.names_file)
|
| 1005 |
+
elif self.num_classes is not None:
|
| 1006 |
+
self.names = [str(i) for i in range(self.num_classes)]
|
| 1007 |
+
else:
|
| 1008 |
+
raise ValueError("Please provide either num_classes, names or names_file.")
|
| 1009 |
+
elif not isinstance(self.names, SequenceABC):
|
| 1010 |
+
raise TypeError(f"Please provide names as a list, is {type(self.names)}")
|
| 1011 |
+
# Set self.num_classes
|
| 1012 |
+
if self.num_classes is None:
|
| 1013 |
+
self.num_classes = len(self.names)
|
| 1014 |
+
elif self.num_classes != len(self.names):
|
| 1015 |
+
raise ValueError(
|
| 1016 |
+
"ClassLabel number of names do not match the defined num_classes. "
|
| 1017 |
+
f"Got {len(self.names)} names VS {self.num_classes} num_classes"
|
| 1018 |
+
)
|
| 1019 |
+
# Prepare mappings
|
| 1020 |
+
self._int2str = [str(name) for name in self.names]
|
| 1021 |
+
self._str2int = {name: i for i, name in enumerate(self._int2str)}
|
| 1022 |
+
if len(self._int2str) != len(self._str2int):
|
| 1023 |
+
raise ValueError("Some label names are duplicated. Each label name should be unique.")
|
| 1024 |
+
|
| 1025 |
+
def __call__(self):
|
| 1026 |
+
return self.pa_type
|
| 1027 |
+
|
| 1028 |
+
def str2int(self, values: Union[str, Iterable]) -> Union[int, Iterable]:
|
| 1029 |
+
"""Conversion class name `string` => `integer`.
|
| 1030 |
+
|
| 1031 |
+
Example:
|
| 1032 |
+
|
| 1033 |
+
```py
|
| 1034 |
+
>>> from datasets import load_dataset
|
| 1035 |
+
>>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
|
| 1036 |
+
>>> ds.features["label"].str2int('neg')
|
| 1037 |
+
0
|
| 1038 |
+
```
|
| 1039 |
+
"""
|
| 1040 |
+
if not isinstance(values, str) and not isinstance(values, Iterable):
|
| 1041 |
+
raise ValueError(
|
| 1042 |
+
f"Values {values} should be a string or an Iterable (list, numpy array, pytorch, tensorflow tensors)"
|
| 1043 |
+
)
|
| 1044 |
+
return_list = True
|
| 1045 |
+
if isinstance(values, str):
|
| 1046 |
+
values = [values]
|
| 1047 |
+
return_list = False
|
| 1048 |
+
|
| 1049 |
+
output = [self._strval2int(value) for value in values]
|
| 1050 |
+
return output if return_list else output[0]
|
| 1051 |
+
|
| 1052 |
+
def _strval2int(self, value: str) -> int:
|
| 1053 |
+
failed_parse = False
|
| 1054 |
+
value = str(value)
|
| 1055 |
+
# first attempt - raw string value
|
| 1056 |
+
int_value = self._str2int.get(value)
|
| 1057 |
+
if int_value is None:
|
| 1058 |
+
# second attempt - strip whitespace
|
| 1059 |
+
int_value = self._str2int.get(value.strip())
|
| 1060 |
+
if int_value is None:
|
| 1061 |
+
# third attempt - convert str to int
|
| 1062 |
+
try:
|
| 1063 |
+
int_value = int(value)
|
| 1064 |
+
except ValueError:
|
| 1065 |
+
failed_parse = True
|
| 1066 |
+
else:
|
| 1067 |
+
if int_value < -1 or int_value >= self.num_classes:
|
| 1068 |
+
failed_parse = True
|
| 1069 |
+
if failed_parse:
|
| 1070 |
+
raise ValueError(f"Invalid string class label {value}")
|
| 1071 |
+
return int_value
|
| 1072 |
+
|
| 1073 |
+
def int2str(self, values: Union[int, Iterable]) -> Union[str, Iterable]:
|
| 1074 |
+
"""Conversion `integer` => class name `string`.
|
| 1075 |
+
|
| 1076 |
+
Regarding unknown/missing labels: passing negative integers raises `ValueError`.
|
| 1077 |
+
|
| 1078 |
+
Example:
|
| 1079 |
+
|
| 1080 |
+
```py
|
| 1081 |
+
>>> from datasets import load_dataset
|
| 1082 |
+
>>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
|
| 1083 |
+
>>> ds.features["label"].int2str(0)
|
| 1084 |
+
'neg'
|
| 1085 |
+
```
|
| 1086 |
+
"""
|
| 1087 |
+
if not isinstance(values, int) and not isinstance(values, Iterable):
|
| 1088 |
+
raise ValueError(
|
| 1089 |
+
f"Values {values} should be an integer or an Iterable (list, numpy array, pytorch, tensorflow tensors)"
|
| 1090 |
+
)
|
| 1091 |
+
return_list = True
|
| 1092 |
+
if isinstance(values, int):
|
| 1093 |
+
values = [values]
|
| 1094 |
+
return_list = False
|
| 1095 |
+
|
| 1096 |
+
for v in values:
|
| 1097 |
+
if not 0 <= v < self.num_classes:
|
| 1098 |
+
raise ValueError(f"Invalid integer class label {v:d}")
|
| 1099 |
+
|
| 1100 |
+
output = [self._int2str[int(v)] for v in values]
|
| 1101 |
+
return output if return_list else output[0]
|
| 1102 |
+
|
| 1103 |
+
def encode_example(self, example_data):
|
| 1104 |
+
if self.num_classes is None:
|
| 1105 |
+
raise ValueError(
|
| 1106 |
+
"Trying to use ClassLabel feature with undefined number of class. "
|
| 1107 |
+
"Please set ClassLabel.names or num_classes."
|
| 1108 |
+
)
|
| 1109 |
+
|
| 1110 |
+
# If a string is given, convert to associated integer
|
| 1111 |
+
if isinstance(example_data, str):
|
| 1112 |
+
example_data = self.str2int(example_data)
|
| 1113 |
+
|
| 1114 |
+
# Allowing -1 to mean no label.
|
| 1115 |
+
if not -1 <= example_data < self.num_classes:
|
| 1116 |
+
raise ValueError(f"Class label {example_data:d} greater than configured num_classes {self.num_classes}")
|
| 1117 |
+
return example_data
|
| 1118 |
+
|
| 1119 |
+
def cast_storage(self, storage: Union[pa.StringArray, pa.IntegerArray]) -> pa.Int64Array:
|
| 1120 |
+
"""Cast an Arrow array to the `ClassLabel` arrow storage type.
|
| 1121 |
+
The Arrow types that can be converted to the `ClassLabel` pyarrow storage type are:
|
| 1122 |
+
|
| 1123 |
+
- `pa.string()`
|
| 1124 |
+
- `pa.int()`
|
| 1125 |
+
|
| 1126 |
+
Args:
|
| 1127 |
+
storage (`Union[pa.StringArray, pa.IntegerArray]`):
|
| 1128 |
+
PyArrow array to cast.
|
| 1129 |
+
|
| 1130 |
+
Returns:
|
| 1131 |
+
`pa.Int64Array`: Array in the `ClassLabel` arrow storage type.
|
| 1132 |
+
"""
|
| 1133 |
+
if isinstance(storage, pa.IntegerArray) and len(storage) > 0:
|
| 1134 |
+
min_max = pc.min_max(storage).as_py()
|
| 1135 |
+
if min_max["max"] is not None and min_max["max"] >= self.num_classes:
|
| 1136 |
+
raise ValueError(
|
| 1137 |
+
f"Class label {min_max['max']} greater than configured num_classes {self.num_classes}"
|
| 1138 |
+
)
|
| 1139 |
+
elif isinstance(storage, pa.StringArray):
|
| 1140 |
+
storage = pa.array(
|
| 1141 |
+
[self._strval2int(label) if label is not None else None for label in storage.to_pylist()]
|
| 1142 |
+
)
|
| 1143 |
+
return array_cast(storage, self.pa_type)
|
| 1144 |
+
|
| 1145 |
+
@staticmethod
|
| 1146 |
+
def _load_names_from_file(names_filepath):
|
| 1147 |
+
with open(names_filepath, encoding="utf-8") as f:
|
| 1148 |
+
return [name.strip() for name in f.read().split("\n") if name.strip()] # Filter empty names
|
| 1149 |
+
|
| 1150 |
+
|
| 1151 |
+
@dataclass
|
| 1152 |
+
class Sequence:
|
| 1153 |
+
"""Construct a list of feature from a single type or a dict of types.
|
| 1154 |
+
Mostly here for compatiblity with tfds.
|
| 1155 |
+
|
| 1156 |
+
Args:
|
| 1157 |
+
feature ([`FeatureType`]):
|
| 1158 |
+
A list of features of a single type or a dictionary of types.
|
| 1159 |
+
length (`int`):
|
| 1160 |
+
Length of the sequence.
|
| 1161 |
+
|
| 1162 |
+
Example:
|
| 1163 |
+
|
| 1164 |
+
```py
|
| 1165 |
+
>>> from datasets import Features, Sequence, Value, ClassLabel
|
| 1166 |
+
>>> features = Features({'post': Sequence(feature={'text': Value(dtype='string'), 'upvotes': Value(dtype='int32'), 'label': ClassLabel(num_classes=2, names=['hot', 'cold'])})})
|
| 1167 |
+
>>> features
|
| 1168 |
+
{'post': Sequence(feature={'text': Value(dtype='string', id=None), 'upvotes': Value(dtype='int32', id=None), 'label': ClassLabel(names=['hot', 'cold'], id=None)}, length=-1, id=None)}
|
| 1169 |
+
```
|
| 1170 |
+
"""
|
| 1171 |
+
|
| 1172 |
+
feature: Any
|
| 1173 |
+
length: int = -1
|
| 1174 |
+
id: Optional[str] = None
|
| 1175 |
+
# Automatically constructed
|
| 1176 |
+
dtype: ClassVar[str] = "list"
|
| 1177 |
+
pa_type: ClassVar[Any] = None
|
| 1178 |
+
_type: str = field(default="Sequence", init=False, repr=False)
|
| 1179 |
+
|
| 1180 |
+
|
| 1181 |
+
@dataclass
|
| 1182 |
+
class LargeList:
|
| 1183 |
+
"""Feature type for large list data composed of child feature data type.
|
| 1184 |
+
|
| 1185 |
+
It is backed by `pyarrow.LargeListType`, which is like `pyarrow.ListType` but with 64-bit rather than 32-bit offsets.
|
| 1186 |
+
|
| 1187 |
+
Args:
|
| 1188 |
+
feature ([`FeatureType`]):
|
| 1189 |
+
Child feature data type of each item within the large list.
|
| 1190 |
+
"""
|
| 1191 |
+
|
| 1192 |
+
feature: Any
|
| 1193 |
+
id: Optional[str] = None
|
| 1194 |
+
# Automatically constructed
|
| 1195 |
+
pa_type: ClassVar[Any] = None
|
| 1196 |
+
_type: str = field(default="LargeList", init=False, repr=False)
|
| 1197 |
+
|
| 1198 |
+
|
| 1199 |
+
FeatureType = Union[
|
| 1200 |
+
dict,
|
| 1201 |
+
list,
|
| 1202 |
+
tuple,
|
| 1203 |
+
Value,
|
| 1204 |
+
ClassLabel,
|
| 1205 |
+
Translation,
|
| 1206 |
+
TranslationVariableLanguages,
|
| 1207 |
+
LargeList,
|
| 1208 |
+
Sequence,
|
| 1209 |
+
Array2D,
|
| 1210 |
+
Array3D,
|
| 1211 |
+
Array4D,
|
| 1212 |
+
Array5D,
|
| 1213 |
+
Audio,
|
| 1214 |
+
Image,
|
| 1215 |
+
Video,
|
| 1216 |
+
Pdf,
|
| 1217 |
+
]
|
| 1218 |
+
|
| 1219 |
+
|
| 1220 |
+
def _check_non_null_non_empty_recursive(obj, schema: Optional[FeatureType] = None) -> bool:
|
| 1221 |
+
"""
|
| 1222 |
+
Check if the object is not None.
|
| 1223 |
+
If the object is a list or a tuple, recursively check the first element of the sequence and stop if at any point the first element is not a sequence or is an empty sequence.
|
| 1224 |
+
"""
|
| 1225 |
+
if obj is None:
|
| 1226 |
+
return False
|
| 1227 |
+
elif isinstance(obj, (list, tuple)) and (schema is None or isinstance(schema, (list, tuple, LargeList, Sequence))):
|
| 1228 |
+
if len(obj) > 0:
|
| 1229 |
+
if schema is None:
|
| 1230 |
+
pass
|
| 1231 |
+
elif isinstance(schema, (list, tuple)):
|
| 1232 |
+
schema = schema[0]
|
| 1233 |
+
else:
|
| 1234 |
+
schema = schema.feature
|
| 1235 |
+
return _check_non_null_non_empty_recursive(obj[0], schema)
|
| 1236 |
+
else:
|
| 1237 |
+
return False
|
| 1238 |
+
else:
|
| 1239 |
+
return True
|
| 1240 |
+
|
| 1241 |
+
|
| 1242 |
+
def get_nested_type(schema: FeatureType) -> pa.DataType:
|
| 1243 |
+
"""
|
| 1244 |
+
get_nested_type() converts a datasets.FeatureType into a pyarrow.DataType, and acts as the inverse of
|
| 1245 |
+
generate_from_arrow_type().
|
| 1246 |
+
|
| 1247 |
+
It performs double-duty as the implementation of Features.type and handles the conversion of
|
| 1248 |
+
datasets.Feature->pa.struct
|
| 1249 |
+
"""
|
| 1250 |
+
# Nested structures: we allow dict, list/tuples, sequences
|
| 1251 |
+
if isinstance(schema, Features):
|
| 1252 |
+
return pa.struct(
|
| 1253 |
+
{key: get_nested_type(schema[key]) for key in schema}
|
| 1254 |
+
) # Features is subclass of dict, and dict order is deterministic since Python 3.6
|
| 1255 |
+
elif isinstance(schema, dict):
|
| 1256 |
+
return pa.struct(
|
| 1257 |
+
{key: get_nested_type(schema[key]) for key in schema}
|
| 1258 |
+
) # however don't sort on struct types since the order matters
|
| 1259 |
+
elif isinstance(schema, (list, tuple)):
|
| 1260 |
+
if len(schema) != 1:
|
| 1261 |
+
raise ValueError("When defining list feature, you should just provide one example of the inner type")
|
| 1262 |
+
value_type = get_nested_type(schema[0])
|
| 1263 |
+
return pa.list_(value_type)
|
| 1264 |
+
elif isinstance(schema, LargeList):
|
| 1265 |
+
value_type = get_nested_type(schema.feature)
|
| 1266 |
+
return pa.large_list(value_type)
|
| 1267 |
+
elif isinstance(schema, Sequence):
|
| 1268 |
+
value_type = get_nested_type(schema.feature)
|
| 1269 |
+
# We allow to reverse list of dict => dict of list for compatibility with tfds
|
| 1270 |
+
if isinstance(schema.feature, dict):
|
| 1271 |
+
data_type = pa.struct({f.name: pa.list_(f.type, schema.length) for f in value_type})
|
| 1272 |
+
else:
|
| 1273 |
+
data_type = pa.list_(value_type, schema.length)
|
| 1274 |
+
return data_type
|
| 1275 |
+
|
| 1276 |
+
# Other objects are callable which returns their data type (ClassLabel, Array2D, Translation, Arrow datatype creation methods)
|
| 1277 |
+
return schema()
|
| 1278 |
+
|
| 1279 |
+
|
| 1280 |
+
def encode_nested_example(schema, obj, level=0):
|
| 1281 |
+
"""Encode a nested example.
|
| 1282 |
+
This is used since some features (in particular ClassLabel) have some logic during encoding.
|
| 1283 |
+
|
| 1284 |
+
To avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be encoded.
|
| 1285 |
+
If the first element needs to be encoded, then all the elements of the list will be encoded, otherwise they'll stay the same.
|
| 1286 |
+
"""
|
| 1287 |
+
# Nested structures: we allow dict, list/tuples, sequences
|
| 1288 |
+
if isinstance(schema, dict):
|
| 1289 |
+
if level == 0 and obj is None:
|
| 1290 |
+
raise ValueError("Got None but expected a dictionary instead")
|
| 1291 |
+
return (
|
| 1292 |
+
{k: encode_nested_example(schema[k], obj.get(k), level=level + 1) for k in schema}
|
| 1293 |
+
if obj is not None
|
| 1294 |
+
else None
|
| 1295 |
+
)
|
| 1296 |
+
|
| 1297 |
+
elif isinstance(schema, (list, tuple)):
|
| 1298 |
+
sub_schema = schema[0]
|
| 1299 |
+
if obj is None:
|
| 1300 |
+
return None
|
| 1301 |
+
elif isinstance(obj, np.ndarray):
|
| 1302 |
+
return encode_nested_example(schema, obj.tolist())
|
| 1303 |
+
else:
|
| 1304 |
+
if len(obj) > 0:
|
| 1305 |
+
for first_elmt in obj:
|
| 1306 |
+
if _check_non_null_non_empty_recursive(first_elmt, sub_schema):
|
| 1307 |
+
break
|
| 1308 |
+
if encode_nested_example(sub_schema, first_elmt, level=level + 1) != first_elmt:
|
| 1309 |
+
return [encode_nested_example(sub_schema, o, level=level + 1) for o in obj]
|
| 1310 |
+
return list(obj)
|
| 1311 |
+
elif isinstance(schema, LargeList):
|
| 1312 |
+
if obj is None:
|
| 1313 |
+
return None
|
| 1314 |
+
else:
|
| 1315 |
+
if len(obj) > 0:
|
| 1316 |
+
sub_schema = schema.feature
|
| 1317 |
+
for first_elmt in obj:
|
| 1318 |
+
if _check_non_null_non_empty_recursive(first_elmt, sub_schema):
|
| 1319 |
+
break
|
| 1320 |
+
if encode_nested_example(sub_schema, first_elmt, level=level + 1) != first_elmt:
|
| 1321 |
+
return [encode_nested_example(sub_schema, o, level=level + 1) for o in obj]
|
| 1322 |
+
return list(obj)
|
| 1323 |
+
elif isinstance(schema, Sequence):
|
| 1324 |
+
if obj is None:
|
| 1325 |
+
return None
|
| 1326 |
+
# We allow to reverse list of dict => dict of list for compatibility with tfds
|
| 1327 |
+
if isinstance(schema.feature, dict):
|
| 1328 |
+
# dict of list to fill
|
| 1329 |
+
list_dict = {}
|
| 1330 |
+
if isinstance(obj, (list, tuple)):
|
| 1331 |
+
# obj is a list of dict
|
| 1332 |
+
for k in schema.feature:
|
| 1333 |
+
list_dict[k] = [encode_nested_example(schema.feature[k], o.get(k), level=level + 1) for o in obj]
|
| 1334 |
+
return list_dict
|
| 1335 |
+
else:
|
| 1336 |
+
# obj is a single dict
|
| 1337 |
+
for k in schema.feature:
|
| 1338 |
+
list_dict[k] = (
|
| 1339 |
+
[encode_nested_example(schema.feature[k], o, level=level + 1) for o in obj[k]]
|
| 1340 |
+
if k in obj
|
| 1341 |
+
else None
|
| 1342 |
+
)
|
| 1343 |
+
return list_dict
|
| 1344 |
+
# schema.feature is not a dict
|
| 1345 |
+
if isinstance(obj, str): # don't interpret a string as a list
|
| 1346 |
+
raise ValueError(f"Got a string but expected a list instead: '{obj}'")
|
| 1347 |
+
else:
|
| 1348 |
+
if len(obj) > 0:
|
| 1349 |
+
for first_elmt in obj:
|
| 1350 |
+
if _check_non_null_non_empty_recursive(first_elmt, schema.feature):
|
| 1351 |
+
break
|
| 1352 |
+
# be careful when comparing tensors here
|
| 1353 |
+
if (
|
| 1354 |
+
not (isinstance(first_elmt, list) or np.isscalar(first_elmt))
|
| 1355 |
+
or encode_nested_example(schema.feature, first_elmt, level=level + 1) != first_elmt
|
| 1356 |
+
):
|
| 1357 |
+
return [encode_nested_example(schema.feature, o, level=level + 1) for o in obj]
|
| 1358 |
+
return list(obj)
|
| 1359 |
+
# Object with special encoding:
|
| 1360 |
+
# ClassLabel will convert from string to int, TranslationVariableLanguages does some checks
|
| 1361 |
+
elif hasattr(schema, "encode_example"):
|
| 1362 |
+
return schema.encode_example(obj) if obj is not None else None
|
| 1363 |
+
# Other object should be directly convertible to a native Arrow type (like Translation and Translation)
|
| 1364 |
+
return obj
|
| 1365 |
+
|
| 1366 |
+
|
| 1367 |
+
def decode_nested_example(schema, obj, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None):
|
| 1368 |
+
"""Decode a nested example.
|
| 1369 |
+
This is used since some features (in particular Audio and Image) have some logic during decoding.
|
| 1370 |
+
|
| 1371 |
+
To avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be decoded.
|
| 1372 |
+
If the first element needs to be decoded, then all the elements of the list will be decoded, otherwise they'll stay the same.
|
| 1373 |
+
"""
|
| 1374 |
+
# Nested structures: we allow dict, list/tuples, sequences
|
| 1375 |
+
if isinstance(schema, dict):
|
| 1376 |
+
return (
|
| 1377 |
+
{k: decode_nested_example(sub_schema, sub_obj) for k, (sub_schema, sub_obj) in zip_dict(schema, obj)}
|
| 1378 |
+
if obj is not None
|
| 1379 |
+
else None
|
| 1380 |
+
)
|
| 1381 |
+
elif isinstance(schema, (list, tuple)):
|
| 1382 |
+
sub_schema = schema[0]
|
| 1383 |
+
if obj is None:
|
| 1384 |
+
return None
|
| 1385 |
+
else:
|
| 1386 |
+
if len(obj) > 0:
|
| 1387 |
+
for first_elmt in obj:
|
| 1388 |
+
if _check_non_null_non_empty_recursive(first_elmt, sub_schema):
|
| 1389 |
+
break
|
| 1390 |
+
if decode_nested_example(sub_schema, first_elmt) != first_elmt:
|
| 1391 |
+
return [decode_nested_example(sub_schema, o) for o in obj]
|
| 1392 |
+
return list(obj)
|
| 1393 |
+
elif isinstance(schema, LargeList):
|
| 1394 |
+
if obj is None:
|
| 1395 |
+
return None
|
| 1396 |
+
else:
|
| 1397 |
+
sub_schema = schema.feature
|
| 1398 |
+
if len(obj) > 0:
|
| 1399 |
+
for first_elmt in obj:
|
| 1400 |
+
if _check_non_null_non_empty_recursive(first_elmt, sub_schema):
|
| 1401 |
+
break
|
| 1402 |
+
if decode_nested_example(sub_schema, first_elmt) != first_elmt:
|
| 1403 |
+
return [decode_nested_example(sub_schema, o) for o in obj]
|
| 1404 |
+
return list(obj)
|
| 1405 |
+
elif isinstance(schema, Sequence):
|
| 1406 |
+
# We allow to reverse list of dict => dict of list for compatibility with tfds
|
| 1407 |
+
if isinstance(schema.feature, dict):
|
| 1408 |
+
return {k: decode_nested_example([schema.feature[k]], obj[k]) for k in schema.feature}
|
| 1409 |
+
else:
|
| 1410 |
+
return decode_nested_example([schema.feature], obj)
|
| 1411 |
+
# Object with special decoding:
|
| 1412 |
+
elif hasattr(schema, "decode_example") and getattr(schema, "decode", True):
|
| 1413 |
+
# we pass the token to read and decode files from private repositories in streaming mode
|
| 1414 |
+
return schema.decode_example(obj, token_per_repo_id=token_per_repo_id) if obj is not None else None
|
| 1415 |
+
return obj
|
| 1416 |
+
|
| 1417 |
+
|
| 1418 |
+
_FEATURE_TYPES: dict[str, FeatureType] = {
|
| 1419 |
+
Value.__name__: Value,
|
| 1420 |
+
ClassLabel.__name__: ClassLabel,
|
| 1421 |
+
Translation.__name__: Translation,
|
| 1422 |
+
TranslationVariableLanguages.__name__: TranslationVariableLanguages,
|
| 1423 |
+
LargeList.__name__: LargeList,
|
| 1424 |
+
Sequence.__name__: Sequence,
|
| 1425 |
+
Array2D.__name__: Array2D,
|
| 1426 |
+
Array3D.__name__: Array3D,
|
| 1427 |
+
Array4D.__name__: Array4D,
|
| 1428 |
+
Array5D.__name__: Array5D,
|
| 1429 |
+
Audio.__name__: Audio,
|
| 1430 |
+
Image.__name__: Image,
|
| 1431 |
+
Video.__name__: Video,
|
| 1432 |
+
Pdf.__name__: Pdf,
|
| 1433 |
+
}
|
| 1434 |
+
|
| 1435 |
+
|
| 1436 |
+
@experimental
|
| 1437 |
+
def register_feature(
|
| 1438 |
+
feature_cls: type,
|
| 1439 |
+
feature_type: str,
|
| 1440 |
+
):
|
| 1441 |
+
"""
|
| 1442 |
+
Register a Feature object using a name and class.
|
| 1443 |
+
This function must be used on a Feature class.
|
| 1444 |
+
"""
|
| 1445 |
+
if feature_type in _FEATURE_TYPES:
|
| 1446 |
+
logger.warning(
|
| 1447 |
+
f"Overwriting feature type '{feature_type}' ({_FEATURE_TYPES[feature_type].__name__} -> {feature_cls.__name__})"
|
| 1448 |
+
)
|
| 1449 |
+
_FEATURE_TYPES[feature_type] = feature_cls
|
| 1450 |
+
|
| 1451 |
+
|
| 1452 |
+
def generate_from_dict(obj: Any):
|
| 1453 |
+
"""Regenerate the nested feature object from a deserialized dict.
|
| 1454 |
+
We use the '_type' fields to get the dataclass name to load.
|
| 1455 |
+
|
| 1456 |
+
generate_from_dict is the recursive helper for Features.from_dict, and allows for a convenient constructor syntax
|
| 1457 |
+
to define features from deserialized JSON dictionaries. This function is used in particular when deserializing
|
| 1458 |
+
a :class:`DatasetInfo` that was dumped to a JSON object. This acts as an analogue to
|
| 1459 |
+
:meth:`Features.from_arrow_schema` and handles the recursive field-by-field instantiation, but doesn't require any
|
| 1460 |
+
mapping to/from pyarrow, except for the fact that it takes advantage of the mapping of pyarrow primitive dtypes
|
| 1461 |
+
that :class:`Value` automatically performs.
|
| 1462 |
+
"""
|
| 1463 |
+
# Nested structures: we allow dict, list/tuples, sequences
|
| 1464 |
+
if isinstance(obj, list):
|
| 1465 |
+
return [generate_from_dict(value) for value in obj]
|
| 1466 |
+
# Otherwise we have a dict or a dataclass
|
| 1467 |
+
if "_type" not in obj or isinstance(obj["_type"], dict):
|
| 1468 |
+
return {key: generate_from_dict(value) for key, value in obj.items()}
|
| 1469 |
+
obj = dict(obj)
|
| 1470 |
+
_type = obj.pop("_type")
|
| 1471 |
+
class_type = _FEATURE_TYPES.get(_type, None) or globals().get(_type, None)
|
| 1472 |
+
|
| 1473 |
+
if class_type is None:
|
| 1474 |
+
raise ValueError(f"Feature type '{_type}' not found. Available feature types: {list(_FEATURE_TYPES.keys())}")
|
| 1475 |
+
|
| 1476 |
+
if class_type == LargeList:
|
| 1477 |
+
feature = obj.pop("feature")
|
| 1478 |
+
return LargeList(feature=generate_from_dict(feature), **obj)
|
| 1479 |
+
if class_type == Sequence:
|
| 1480 |
+
feature = obj.pop("feature")
|
| 1481 |
+
return Sequence(feature=generate_from_dict(feature), **obj)
|
| 1482 |
+
|
| 1483 |
+
field_names = {f.name for f in fields(class_type)}
|
| 1484 |
+
return class_type(**{k: v for k, v in obj.items() if k in field_names})
|
| 1485 |
+
|
| 1486 |
+
|
| 1487 |
+
def generate_from_arrow_type(pa_type: pa.DataType) -> FeatureType:
|
| 1488 |
+
"""
|
| 1489 |
+
generate_from_arrow_type accepts an arrow DataType and returns a datasets FeatureType to be used as the type for
|
| 1490 |
+
a single field.
|
| 1491 |
+
|
| 1492 |
+
This is the high-level arrow->datasets type conversion and is inverted by get_nested_type().
|
| 1493 |
+
|
| 1494 |
+
This operates at the individual *field* level, whereas Features.from_arrow_schema() operates at the
|
| 1495 |
+
full schema level and holds the methods that represent the bijection from Features<->pyarrow.Schema
|
| 1496 |
+
"""
|
| 1497 |
+
if isinstance(pa_type, pa.StructType):
|
| 1498 |
+
return {field.name: generate_from_arrow_type(field.type) for field in pa_type}
|
| 1499 |
+
elif isinstance(pa_type, pa.FixedSizeListType):
|
| 1500 |
+
return Sequence(feature=generate_from_arrow_type(pa_type.value_type), length=pa_type.list_size)
|
| 1501 |
+
elif isinstance(pa_type, pa.ListType):
|
| 1502 |
+
feature = generate_from_arrow_type(pa_type.value_type)
|
| 1503 |
+
if isinstance(feature, (dict, tuple, list)):
|
| 1504 |
+
return [feature]
|
| 1505 |
+
return Sequence(feature=feature)
|
| 1506 |
+
elif isinstance(pa_type, pa.LargeListType):
|
| 1507 |
+
feature = generate_from_arrow_type(pa_type.value_type)
|
| 1508 |
+
return LargeList(feature=feature)
|
| 1509 |
+
elif isinstance(pa_type, _ArrayXDExtensionType):
|
| 1510 |
+
array_feature = [None, None, Array2D, Array3D, Array4D, Array5D][pa_type.ndims]
|
| 1511 |
+
return array_feature(shape=pa_type.shape, dtype=pa_type.value_type)
|
| 1512 |
+
elif isinstance(pa_type, pa.DataType):
|
| 1513 |
+
return Value(dtype=_arrow_to_datasets_dtype(pa_type))
|
| 1514 |
+
else:
|
| 1515 |
+
raise ValueError(f"Cannot convert {pa_type} to a Feature type.")
|
| 1516 |
+
|
| 1517 |
+
|
| 1518 |
+
def numpy_to_pyarrow_listarray(arr: np.ndarray, type: pa.DataType = None) -> pa.ListArray:
|
| 1519 |
+
"""Build a PyArrow ListArray from a multidimensional NumPy array"""
|
| 1520 |
+
arr = np.array(arr)
|
| 1521 |
+
values = pa.array(arr.flatten(), type=type)
|
| 1522 |
+
for i in range(arr.ndim - 1):
|
| 1523 |
+
n_offsets = reduce(mul, arr.shape[: arr.ndim - i - 1], 1)
|
| 1524 |
+
step_offsets = arr.shape[arr.ndim - i - 1]
|
| 1525 |
+
offsets = pa.array(np.arange(n_offsets + 1) * step_offsets, type=pa.int32())
|
| 1526 |
+
values = pa.ListArray.from_arrays(offsets, values)
|
| 1527 |
+
return values
|
| 1528 |
+
|
| 1529 |
+
|
| 1530 |
+
def list_of_pa_arrays_to_pyarrow_listarray(l_arr: list[Optional[pa.Array]]) -> pa.ListArray:
|
| 1531 |
+
null_mask = np.array([arr is None for arr in l_arr])
|
| 1532 |
+
null_indices = np.arange(len(null_mask))[null_mask] - np.arange(np.sum(null_mask))
|
| 1533 |
+
l_arr = [arr for arr in l_arr if arr is not None]
|
| 1534 |
+
offsets = np.cumsum(
|
| 1535 |
+
[0] + [len(arr) for arr in l_arr], dtype=object
|
| 1536 |
+
) # convert to dtype object to allow None insertion
|
| 1537 |
+
offsets = np.insert(offsets, null_indices, None)
|
| 1538 |
+
offsets = pa.array(offsets, type=pa.int32())
|
| 1539 |
+
values = pa.concat_arrays(l_arr)
|
| 1540 |
+
return pa.ListArray.from_arrays(offsets, values)
|
| 1541 |
+
|
| 1542 |
+
|
| 1543 |
+
def list_of_np_array_to_pyarrow_listarray(l_arr: list[np.ndarray], type: pa.DataType = None) -> pa.ListArray:
|
| 1544 |
+
"""Build a PyArrow ListArray from a possibly nested list of NumPy arrays"""
|
| 1545 |
+
if len(l_arr) > 0:
|
| 1546 |
+
return list_of_pa_arrays_to_pyarrow_listarray(
|
| 1547 |
+
[numpy_to_pyarrow_listarray(arr, type=type) if arr is not None else None for arr in l_arr]
|
| 1548 |
+
)
|
| 1549 |
+
else:
|
| 1550 |
+
return pa.array([], type=type)
|
| 1551 |
+
|
| 1552 |
+
|
| 1553 |
+
def contains_any_np_array(data: Any):
|
| 1554 |
+
"""Return `True` if data is a NumPy ndarray or (recursively) if first non-null value in list is a NumPy ndarray.
|
| 1555 |
+
|
| 1556 |
+
Args:
|
| 1557 |
+
data (Any): Data.
|
| 1558 |
+
|
| 1559 |
+
Returns:
|
| 1560 |
+
bool
|
| 1561 |
+
"""
|
| 1562 |
+
if isinstance(data, np.ndarray):
|
| 1563 |
+
return True
|
| 1564 |
+
elif isinstance(data, list):
|
| 1565 |
+
return contains_any_np_array(first_non_null_value(data)[1])
|
| 1566 |
+
else:
|
| 1567 |
+
return False
|
| 1568 |
+
|
| 1569 |
+
|
| 1570 |
+
def any_np_array_to_pyarrow_listarray(data: Union[np.ndarray, list], type: pa.DataType = None) -> pa.ListArray:
|
| 1571 |
+
"""Convert to PyArrow ListArray either a NumPy ndarray or (recursively) a list that may contain any NumPy ndarray.
|
| 1572 |
+
|
| 1573 |
+
Args:
|
| 1574 |
+
data (Union[np.ndarray, List]): Data.
|
| 1575 |
+
type (pa.DataType): Explicit PyArrow DataType passed to coerce the ListArray data type.
|
| 1576 |
+
|
| 1577 |
+
Returns:
|
| 1578 |
+
pa.ListArray
|
| 1579 |
+
"""
|
| 1580 |
+
if isinstance(data, np.ndarray):
|
| 1581 |
+
return numpy_to_pyarrow_listarray(data, type=type)
|
| 1582 |
+
elif isinstance(data, list):
|
| 1583 |
+
return list_of_pa_arrays_to_pyarrow_listarray([any_np_array_to_pyarrow_listarray(i, type=type) for i in data])
|
| 1584 |
+
|
| 1585 |
+
|
| 1586 |
+
def to_pyarrow_listarray(data: Any, pa_type: _ArrayXDExtensionType) -> pa.Array:
|
| 1587 |
+
"""Convert to PyArrow ListArray.
|
| 1588 |
+
|
| 1589 |
+
Args:
|
| 1590 |
+
data (Any): Sequence, iterable, np.ndarray or pd.Series.
|
| 1591 |
+
pa_type (_ArrayXDExtensionType): Any of the ArrayNDExtensionType.
|
| 1592 |
+
|
| 1593 |
+
Returns:
|
| 1594 |
+
pyarrow.Array
|
| 1595 |
+
"""
|
| 1596 |
+
if contains_any_np_array(data):
|
| 1597 |
+
return any_np_array_to_pyarrow_listarray(data, type=pa_type.value_type)
|
| 1598 |
+
else:
|
| 1599 |
+
return pa.array(data, pa_type.storage_dtype)
|
| 1600 |
+
|
| 1601 |
+
|
| 1602 |
+
def _visit(feature: FeatureType, func: Callable[[FeatureType], Optional[FeatureType]]) -> FeatureType:
|
| 1603 |
+
"""Visit a (possibly nested) feature.
|
| 1604 |
+
|
| 1605 |
+
Args:
|
| 1606 |
+
feature (FeatureType): the feature type to be checked
|
| 1607 |
+
Returns:
|
| 1608 |
+
visited feature (FeatureType)
|
| 1609 |
+
"""
|
| 1610 |
+
if isinstance(feature, Features):
|
| 1611 |
+
out = func(Features({k: _visit(f, func) for k, f in feature.items()}))
|
| 1612 |
+
elif isinstance(feature, dict):
|
| 1613 |
+
out = func({k: _visit(f, func) for k, f in feature.items()})
|
| 1614 |
+
elif isinstance(feature, (list, tuple)):
|
| 1615 |
+
out = func([_visit(feature[0], func)])
|
| 1616 |
+
elif isinstance(feature, LargeList):
|
| 1617 |
+
out = func(LargeList(_visit(feature.feature, func)))
|
| 1618 |
+
elif isinstance(feature, Sequence):
|
| 1619 |
+
out = func(Sequence(_visit(feature.feature, func), length=feature.length))
|
| 1620 |
+
else:
|
| 1621 |
+
out = func(feature)
|
| 1622 |
+
return feature if out is None else out
|
| 1623 |
+
|
| 1624 |
+
|
| 1625 |
+
_VisitPath = list[Union[str, Literal[0]]]
|
| 1626 |
+
|
| 1627 |
+
|
| 1628 |
+
def _visit_with_path(
|
| 1629 |
+
feature: FeatureType, func: Callable[[FeatureType, _VisitPath], Optional[FeatureType]], visit_path: _VisitPath = []
|
| 1630 |
+
) -> FeatureType:
|
| 1631 |
+
"""Visit a (possibly nested) feature with its path in the Feature object.
|
| 1632 |
+
|
| 1633 |
+
A path in a nested feature object is the list of keys that need to be
|
| 1634 |
+
sequentially accessed to get to the sub-feature.
|
| 1635 |
+
|
| 1636 |
+
For example:
|
| 1637 |
+
- ["foo"] corresponds to the column "foo"
|
| 1638 |
+
- ["foo", 0] corresponds to the sub-feature of the lists in "foo"
|
| 1639 |
+
- ["foo", "bar"] corresponds to the sub-feature of the dicts in "foo" with key "bar"
|
| 1640 |
+
|
| 1641 |
+
Args:
|
| 1642 |
+
feature (`FeatureType`): the feature type to be checked.
|
| 1643 |
+
|
| 1644 |
+
Returns:
|
| 1645 |
+
`FeatureType`: the visited feature.
|
| 1646 |
+
"""
|
| 1647 |
+
if isinstance(feature, Sequence) and isinstance(feature.feature, dict):
|
| 1648 |
+
feature = {k: [f] for k, f in feature.feature.items()}
|
| 1649 |
+
# ^ Sequence of dicts is special, it must be converted to a dict of lists (see https://huggingface.co/docs/datasets/v2.16.1/en/package_reference/main_classes#datasets.Features)
|
| 1650 |
+
if isinstance(feature, Features):
|
| 1651 |
+
out = func(Features({k: _visit_with_path(f, func, visit_path + [k]) for k, f in feature.items()}), visit_path)
|
| 1652 |
+
elif isinstance(feature, dict):
|
| 1653 |
+
out = func({k: _visit_with_path(f, func, visit_path + [k]) for k, f in feature.items()}, visit_path)
|
| 1654 |
+
elif isinstance(feature, (list, tuple)):
|
| 1655 |
+
out = func([_visit_with_path(feature[0], func, visit_path + [0])], visit_path)
|
| 1656 |
+
elif isinstance(feature, Sequence):
|
| 1657 |
+
out = func(
|
| 1658 |
+
Sequence(_visit_with_path(feature.feature, func, visit_path + [0]), length=feature.length), visit_path
|
| 1659 |
+
)
|
| 1660 |
+
elif isinstance(feature, LargeList):
|
| 1661 |
+
out = func(LargeList(_visit_with_path(feature.feature, func, visit_path + [0])), visit_path)
|
| 1662 |
+
else:
|
| 1663 |
+
out = func(feature, visit_path)
|
| 1664 |
+
return feature if out is None else out
|
| 1665 |
+
|
| 1666 |
+
|
| 1667 |
+
def require_decoding(feature: FeatureType, ignore_decode_attribute: bool = False) -> bool:
|
| 1668 |
+
"""Check if a (possibly nested) feature requires decoding.
|
| 1669 |
+
|
| 1670 |
+
Args:
|
| 1671 |
+
feature (FeatureType): the feature type to be checked
|
| 1672 |
+
ignore_decode_attribute (:obj:`bool`, default ``False``): Whether to ignore the current value
|
| 1673 |
+
of the `decode` attribute of the decodable feature types.
|
| 1674 |
+
Returns:
|
| 1675 |
+
:obj:`bool`
|
| 1676 |
+
"""
|
| 1677 |
+
if isinstance(feature, dict):
|
| 1678 |
+
return any(require_decoding(f) for f in feature.values())
|
| 1679 |
+
elif isinstance(feature, (list, tuple)):
|
| 1680 |
+
return require_decoding(feature[0])
|
| 1681 |
+
elif isinstance(feature, LargeList):
|
| 1682 |
+
return require_decoding(feature.feature)
|
| 1683 |
+
elif isinstance(feature, Sequence):
|
| 1684 |
+
return require_decoding(feature.feature)
|
| 1685 |
+
else:
|
| 1686 |
+
return hasattr(feature, "decode_example") and (
|
| 1687 |
+
getattr(feature, "decode", True) if not ignore_decode_attribute else True
|
| 1688 |
+
)
|
| 1689 |
+
|
| 1690 |
+
|
| 1691 |
+
def require_storage_cast(feature: FeatureType) -> bool:
|
| 1692 |
+
"""Check if a (possibly nested) feature requires storage casting.
|
| 1693 |
+
|
| 1694 |
+
Args:
|
| 1695 |
+
feature (FeatureType): the feature type to be checked
|
| 1696 |
+
Returns:
|
| 1697 |
+
:obj:`bool`
|
| 1698 |
+
"""
|
| 1699 |
+
if isinstance(feature, dict):
|
| 1700 |
+
return any(require_storage_cast(f) for f in feature.values())
|
| 1701 |
+
elif isinstance(feature, (list, tuple)):
|
| 1702 |
+
return require_storage_cast(feature[0])
|
| 1703 |
+
elif isinstance(feature, LargeList):
|
| 1704 |
+
return require_storage_cast(feature.feature)
|
| 1705 |
+
elif isinstance(feature, Sequence):
|
| 1706 |
+
return require_storage_cast(feature.feature)
|
| 1707 |
+
else:
|
| 1708 |
+
return hasattr(feature, "cast_storage")
|
| 1709 |
+
|
| 1710 |
+
|
| 1711 |
+
def require_storage_embed(feature: FeatureType) -> bool:
|
| 1712 |
+
"""Check if a (possibly nested) feature requires embedding data into storage.
|
| 1713 |
+
|
| 1714 |
+
Args:
|
| 1715 |
+
feature (FeatureType): the feature type to be checked
|
| 1716 |
+
Returns:
|
| 1717 |
+
:obj:`bool`
|
| 1718 |
+
"""
|
| 1719 |
+
if isinstance(feature, dict):
|
| 1720 |
+
return any(require_storage_cast(f) for f in feature.values())
|
| 1721 |
+
elif isinstance(feature, (list, tuple)):
|
| 1722 |
+
return require_storage_cast(feature[0])
|
| 1723 |
+
elif isinstance(feature, LargeList):
|
| 1724 |
+
return require_storage_cast(feature.feature)
|
| 1725 |
+
elif isinstance(feature, Sequence):
|
| 1726 |
+
return require_storage_cast(feature.feature)
|
| 1727 |
+
else:
|
| 1728 |
+
return hasattr(feature, "embed_storage")
|
| 1729 |
+
|
| 1730 |
+
|
| 1731 |
+
def keep_features_dicts_synced(func):
|
| 1732 |
+
"""
|
| 1733 |
+
Wrapper to keep the secondary dictionary, which tracks whether keys are decodable, of the :class:`datasets.Features` object
|
| 1734 |
+
in sync with the main dictionary.
|
| 1735 |
+
"""
|
| 1736 |
+
|
| 1737 |
+
@wraps(func)
|
| 1738 |
+
def wrapper(*args, **kwargs):
|
| 1739 |
+
if args:
|
| 1740 |
+
self: "Features" = args[0]
|
| 1741 |
+
args = args[1:]
|
| 1742 |
+
else:
|
| 1743 |
+
self: "Features" = kwargs.pop("self")
|
| 1744 |
+
out = func(self, *args, **kwargs)
|
| 1745 |
+
assert hasattr(self, "_column_requires_decoding")
|
| 1746 |
+
self._column_requires_decoding = {col: require_decoding(feature) for col, feature in self.items()}
|
| 1747 |
+
return out
|
| 1748 |
+
|
| 1749 |
+
wrapper._decorator_name_ = "_keep_dicts_synced"
|
| 1750 |
+
return wrapper
|
| 1751 |
+
|
| 1752 |
+
|
| 1753 |
+
class Features(dict):
|
| 1754 |
+
"""A special dictionary that defines the internal structure of a dataset.
|
| 1755 |
+
|
| 1756 |
+
Instantiated with a dictionary of type `dict[str, FieldType]`, where keys are the desired column names,
|
| 1757 |
+
and values are the type of that column.
|
| 1758 |
+
|
| 1759 |
+
`FieldType` can be one of the following:
|
| 1760 |
+
- [`Value`] feature specifies a single data type value, e.g. `int64` or `string`.
|
| 1761 |
+
- [`ClassLabel`] feature specifies a predefined set of classes which can have labels associated to them and
|
| 1762 |
+
will be stored as integers in the dataset.
|
| 1763 |
+
- Python `dict` specifies a composite feature containing a mapping of sub-fields to sub-features.
|
| 1764 |
+
It's possible to have nested fields of nested fields in an arbitrary manner.
|
| 1765 |
+
- Python `list`, [`LargeList`] or [`Sequence`] specifies a composite feature containing a sequence of
|
| 1766 |
+
sub-features, all of the same feature type.
|
| 1767 |
+
|
| 1768 |
+
<Tip>
|
| 1769 |
+
|
| 1770 |
+
A [`Sequence`] with an internal dictionary feature will be automatically converted into a dictionary of
|
| 1771 |
+
lists. This behavior is implemented to have a compatibility layer with the TensorFlow Datasets library but may be
|
| 1772 |
+
un-wanted in some cases. If you don't want this behavior, you can use a Python `list` or a [`LargeList`]
|
| 1773 |
+
instead of the [`Sequence`].
|
| 1774 |
+
|
| 1775 |
+
</Tip>
|
| 1776 |
+
|
| 1777 |
+
- [`Array2D`], [`Array3D`], [`Array4D`] or [`Array5D`] feature for multidimensional arrays.
|
| 1778 |
+
- [`Audio`] feature to store the absolute path to an audio file or a dictionary with the relative path
|
| 1779 |
+
to an audio file ("path" key) and its bytes content ("bytes" key). This feature extracts the audio data.
|
| 1780 |
+
- [`Image`] feature to store the absolute path to an image file, an `np.ndarray` object, a `PIL.Image.Image` object
|
| 1781 |
+
or a dictionary with the relative path to an image file ("path" key) and its bytes content ("bytes" key).
|
| 1782 |
+
This feature extracts the image data.
|
| 1783 |
+
- [`Translation`] or [`TranslationVariableLanguages`] feature specific to Machine Translation.
|
| 1784 |
+
"""
|
| 1785 |
+
|
| 1786 |
+
def __init__(*args, **kwargs):
|
| 1787 |
+
# self not in the signature to allow passing self as a kwarg
|
| 1788 |
+
if not args:
|
| 1789 |
+
raise TypeError("descriptor '__init__' of 'Features' object needs an argument")
|
| 1790 |
+
self, *args = args
|
| 1791 |
+
super(Features, self).__init__(*args, **kwargs)
|
| 1792 |
+
self._column_requires_decoding: dict[str, bool] = {
|
| 1793 |
+
col: require_decoding(feature) for col, feature in self.items()
|
| 1794 |
+
}
|
| 1795 |
+
|
| 1796 |
+
__setitem__ = keep_features_dicts_synced(dict.__setitem__)
|
| 1797 |
+
__delitem__ = keep_features_dicts_synced(dict.__delitem__)
|
| 1798 |
+
update = keep_features_dicts_synced(dict.update)
|
| 1799 |
+
setdefault = keep_features_dicts_synced(dict.setdefault)
|
| 1800 |
+
pop = keep_features_dicts_synced(dict.pop)
|
| 1801 |
+
popitem = keep_features_dicts_synced(dict.popitem)
|
| 1802 |
+
clear = keep_features_dicts_synced(dict.clear)
|
| 1803 |
+
|
| 1804 |
+
def __reduce__(self):
|
| 1805 |
+
return Features, (dict(self),)
|
| 1806 |
+
|
| 1807 |
+
@property
|
| 1808 |
+
def type(self):
|
| 1809 |
+
"""
|
| 1810 |
+
Features field types.
|
| 1811 |
+
|
| 1812 |
+
Returns:
|
| 1813 |
+
:obj:`pyarrow.DataType`
|
| 1814 |
+
"""
|
| 1815 |
+
return get_nested_type(self)
|
| 1816 |
+
|
| 1817 |
+
@property
|
| 1818 |
+
def arrow_schema(self):
|
| 1819 |
+
"""
|
| 1820 |
+
Features schema.
|
| 1821 |
+
|
| 1822 |
+
Returns:
|
| 1823 |
+
:obj:`pyarrow.Schema`
|
| 1824 |
+
"""
|
| 1825 |
+
hf_metadata = {"info": {"features": self.to_dict()}}
|
| 1826 |
+
return pa.schema(self.type).with_metadata({"huggingface": json.dumps(hf_metadata)})
|
| 1827 |
+
|
| 1828 |
+
@classmethod
|
| 1829 |
+
def from_arrow_schema(cls, pa_schema: pa.Schema) -> "Features":
|
| 1830 |
+
"""
|
| 1831 |
+
Construct [`Features`] from Arrow Schema.
|
| 1832 |
+
It also checks the schema metadata for Hugging Face Datasets features.
|
| 1833 |
+
Non-nullable fields are not supported and set to nullable.
|
| 1834 |
+
|
| 1835 |
+
Also, pa.dictionary is not supported and it uses its underlying type instead.
|
| 1836 |
+
Therefore datasets convert DictionaryArray objects to their actual values.
|
| 1837 |
+
|
| 1838 |
+
Args:
|
| 1839 |
+
pa_schema (`pyarrow.Schema`):
|
| 1840 |
+
Arrow Schema.
|
| 1841 |
+
|
| 1842 |
+
Returns:
|
| 1843 |
+
[`Features`]
|
| 1844 |
+
"""
|
| 1845 |
+
# try to load features from the arrow schema metadata
|
| 1846 |
+
metadata_features = Features()
|
| 1847 |
+
if pa_schema.metadata is not None and b"huggingface" in pa_schema.metadata:
|
| 1848 |
+
metadata = json.loads(pa_schema.metadata[b"huggingface"].decode())
|
| 1849 |
+
if "info" in metadata and "features" in metadata["info"] and metadata["info"]["features"] is not None:
|
| 1850 |
+
metadata_features = Features.from_dict(metadata["info"]["features"])
|
| 1851 |
+
metadata_features_schema = metadata_features.arrow_schema
|
| 1852 |
+
obj = {
|
| 1853 |
+
field.name: (
|
| 1854 |
+
metadata_features[field.name]
|
| 1855 |
+
if field.name in metadata_features and metadata_features_schema.field(field.name) == field
|
| 1856 |
+
else generate_from_arrow_type(field.type)
|
| 1857 |
+
)
|
| 1858 |
+
for field in pa_schema
|
| 1859 |
+
}
|
| 1860 |
+
return cls(**obj)
|
| 1861 |
+
|
| 1862 |
+
@classmethod
|
| 1863 |
+
def from_dict(cls, dic) -> "Features":
|
| 1864 |
+
"""
|
| 1865 |
+
Construct [`Features`] from dict.
|
| 1866 |
+
|
| 1867 |
+
Regenerate the nested feature object from a deserialized dict.
|
| 1868 |
+
We use the `_type` key to infer the dataclass name of the feature `FieldType`.
|
| 1869 |
+
|
| 1870 |
+
It allows for a convenient constructor syntax
|
| 1871 |
+
to define features from deserialized JSON dictionaries. This function is used in particular when deserializing
|
| 1872 |
+
a [`DatasetInfo`] that was dumped to a JSON object. This acts as an analogue to
|
| 1873 |
+
[`Features.from_arrow_schema`] and handles the recursive field-by-field instantiation, but doesn't require
|
| 1874 |
+
any mapping to/from pyarrow, except for the fact that it takes advantage of the mapping of pyarrow primitive
|
| 1875 |
+
dtypes that [`Value`] automatically performs.
|
| 1876 |
+
|
| 1877 |
+
Args:
|
| 1878 |
+
dic (`dict[str, Any]`):
|
| 1879 |
+
Python dictionary.
|
| 1880 |
+
|
| 1881 |
+
Returns:
|
| 1882 |
+
`Features`
|
| 1883 |
+
|
| 1884 |
+
Example::
|
| 1885 |
+
>>> Features.from_dict({'_type': {'dtype': 'string', 'id': None, '_type': 'Value'}})
|
| 1886 |
+
{'_type': Value(dtype='string', id=None)}
|
| 1887 |
+
"""
|
| 1888 |
+
obj = generate_from_dict(dic)
|
| 1889 |
+
return cls(**obj)
|
| 1890 |
+
|
| 1891 |
+
def to_dict(self):
|
| 1892 |
+
return asdict(self)
|
| 1893 |
+
|
| 1894 |
+
def _to_yaml_list(self) -> list:
|
| 1895 |
+
# we compute the YAML list from the dict representation that is used for JSON dump
|
| 1896 |
+
yaml_data = self.to_dict()
|
| 1897 |
+
|
| 1898 |
+
def simplify(feature: dict) -> dict:
|
| 1899 |
+
if not isinstance(feature, dict):
|
| 1900 |
+
raise TypeError(f"Expected a dict but got a {type(feature)}: {feature}")
|
| 1901 |
+
|
| 1902 |
+
for list_type in ["large_list", "list", "sequence"]:
|
| 1903 |
+
#
|
| 1904 |
+
# list_type: -> list_type: int32
|
| 1905 |
+
# dtype: int32 ->
|
| 1906 |
+
#
|
| 1907 |
+
if isinstance(feature.get(list_type), dict) and list(feature[list_type]) == ["dtype"]:
|
| 1908 |
+
feature[list_type] = feature[list_type]["dtype"]
|
| 1909 |
+
|
| 1910 |
+
#
|
| 1911 |
+
# list_type: -> list_type:
|
| 1912 |
+
# struct: -> - name: foo
|
| 1913 |
+
# - name: foo -> dtype: int32
|
| 1914 |
+
# dtype: int32 ->
|
| 1915 |
+
#
|
| 1916 |
+
if isinstance(feature.get(list_type), dict) and list(feature[list_type]) == ["struct"]:
|
| 1917 |
+
feature[list_type] = feature[list_type]["struct"]
|
| 1918 |
+
|
| 1919 |
+
#
|
| 1920 |
+
# class_label: -> class_label:
|
| 1921 |
+
# names: -> names:
|
| 1922 |
+
# - negative -> '0': negative
|
| 1923 |
+
# - positive -> '1': positive
|
| 1924 |
+
#
|
| 1925 |
+
if isinstance(feature.get("class_label"), dict) and isinstance(feature["class_label"].get("names"), list):
|
| 1926 |
+
# server-side requirement: keys must be strings
|
| 1927 |
+
feature["class_label"]["names"] = {
|
| 1928 |
+
str(label_id): label_name for label_id, label_name in enumerate(feature["class_label"]["names"])
|
| 1929 |
+
}
|
| 1930 |
+
return feature
|
| 1931 |
+
|
| 1932 |
+
def to_yaml_inner(obj: Union[dict, list]) -> dict:
|
| 1933 |
+
if isinstance(obj, dict):
|
| 1934 |
+
_type = obj.pop("_type", None)
|
| 1935 |
+
if _type == "LargeList":
|
| 1936 |
+
_feature = obj.pop("feature")
|
| 1937 |
+
return simplify({"large_list": to_yaml_inner(_feature), **obj})
|
| 1938 |
+
elif _type == "Sequence":
|
| 1939 |
+
_feature = obj.pop("feature")
|
| 1940 |
+
return simplify({"sequence": to_yaml_inner(_feature), **obj})
|
| 1941 |
+
elif _type == "Value":
|
| 1942 |
+
return obj
|
| 1943 |
+
elif _type and not obj:
|
| 1944 |
+
return {"dtype": camelcase_to_snakecase(_type)}
|
| 1945 |
+
elif _type:
|
| 1946 |
+
return {"dtype": simplify({camelcase_to_snakecase(_type): obj})}
|
| 1947 |
+
else:
|
| 1948 |
+
return {"struct": [{"name": name, **to_yaml_inner(_feature)} for name, _feature in obj.items()]}
|
| 1949 |
+
elif isinstance(obj, list):
|
| 1950 |
+
return simplify({"list": simplify(to_yaml_inner(obj[0]))})
|
| 1951 |
+
elif isinstance(obj, tuple):
|
| 1952 |
+
return to_yaml_inner(list(obj))
|
| 1953 |
+
else:
|
| 1954 |
+
raise TypeError(f"Expected a dict or a list but got {type(obj)}: {obj}")
|
| 1955 |
+
|
| 1956 |
+
def to_yaml_types(obj: dict) -> dict:
|
| 1957 |
+
if isinstance(obj, dict):
|
| 1958 |
+
return {k: to_yaml_types(v) for k, v in obj.items()}
|
| 1959 |
+
elif isinstance(obj, list):
|
| 1960 |
+
return [to_yaml_types(v) for v in obj]
|
| 1961 |
+
elif isinstance(obj, tuple):
|
| 1962 |
+
return to_yaml_types(list(obj))
|
| 1963 |
+
else:
|
| 1964 |
+
return obj
|
| 1965 |
+
|
| 1966 |
+
return to_yaml_types(to_yaml_inner(yaml_data)["struct"])
|
| 1967 |
+
|
| 1968 |
+
@classmethod
|
| 1969 |
+
def _from_yaml_list(cls, yaml_data: list) -> "Features":
|
| 1970 |
+
yaml_data = copy.deepcopy(yaml_data)
|
| 1971 |
+
|
| 1972 |
+
# we convert the list obtained from YAML data into the dict representation that is used for JSON dump
|
| 1973 |
+
|
| 1974 |
+
def unsimplify(feature: dict) -> dict:
|
| 1975 |
+
if not isinstance(feature, dict):
|
| 1976 |
+
raise TypeError(f"Expected a dict but got a {type(feature)}: {feature}")
|
| 1977 |
+
|
| 1978 |
+
for list_type in ["large_list", "list", "sequence"]:
|
| 1979 |
+
#
|
| 1980 |
+
# list_type: int32 -> list_type:
|
| 1981 |
+
# -> dtype: int32
|
| 1982 |
+
#
|
| 1983 |
+
if isinstance(feature.get(list_type), str):
|
| 1984 |
+
feature[list_type] = {"dtype": feature[list_type]}
|
| 1985 |
+
|
| 1986 |
+
#
|
| 1987 |
+
# class_label: -> class_label:
|
| 1988 |
+
# names: -> names:
|
| 1989 |
+
# '0': negative -> - negative
|
| 1990 |
+
# '1': positive -> - positive
|
| 1991 |
+
#
|
| 1992 |
+
if isinstance(feature.get("class_label"), dict) and isinstance(feature["class_label"].get("names"), dict):
|
| 1993 |
+
label_ids = sorted(feature["class_label"]["names"], key=int)
|
| 1994 |
+
if label_ids and [int(label_id) for label_id in label_ids] != list(range(int(label_ids[-1]) + 1)):
|
| 1995 |
+
raise ValueError(
|
| 1996 |
+
f"ClassLabel expected a value for all label ids [0:{int(label_ids[-1]) + 1}] but some ids are missing."
|
| 1997 |
+
)
|
| 1998 |
+
feature["class_label"]["names"] = [feature["class_label"]["names"][label_id] for label_id in label_ids]
|
| 1999 |
+
return feature
|
| 2000 |
+
|
| 2001 |
+
def from_yaml_inner(obj: Union[dict, list]) -> Union[dict, list]:
|
| 2002 |
+
if isinstance(obj, dict):
|
| 2003 |
+
if not obj:
|
| 2004 |
+
return {}
|
| 2005 |
+
_type = next(iter(obj))
|
| 2006 |
+
if _type == "large_list":
|
| 2007 |
+
_feature = unsimplify(obj).pop(_type)
|
| 2008 |
+
return {"feature": from_yaml_inner(_feature), **obj, "_type": "LargeList"}
|
| 2009 |
+
if _type == "sequence":
|
| 2010 |
+
_feature = unsimplify(obj).pop(_type)
|
| 2011 |
+
return {"feature": from_yaml_inner(_feature), **obj, "_type": "Sequence"}
|
| 2012 |
+
if _type == "list":
|
| 2013 |
+
return [from_yaml_inner(unsimplify(obj)[_type])]
|
| 2014 |
+
if _type == "struct":
|
| 2015 |
+
return from_yaml_inner(obj["struct"])
|
| 2016 |
+
elif _type == "dtype":
|
| 2017 |
+
if isinstance(obj["dtype"], str):
|
| 2018 |
+
# e.g. int32, float64, string, audio, image
|
| 2019 |
+
try:
|
| 2020 |
+
Value(obj["dtype"])
|
| 2021 |
+
return {**obj, "_type": "Value"}
|
| 2022 |
+
except ValueError:
|
| 2023 |
+
# e.g. Audio, Image, ArrayXD
|
| 2024 |
+
return {"_type": snakecase_to_camelcase(obj["dtype"])}
|
| 2025 |
+
else:
|
| 2026 |
+
return from_yaml_inner(obj["dtype"])
|
| 2027 |
+
else:
|
| 2028 |
+
return {"_type": snakecase_to_camelcase(_type), **unsimplify(obj)[_type]}
|
| 2029 |
+
elif isinstance(obj, list):
|
| 2030 |
+
names = [_feature.pop("name") for _feature in obj]
|
| 2031 |
+
return {name: from_yaml_inner(_feature) for name, _feature in zip(names, obj)}
|
| 2032 |
+
else:
|
| 2033 |
+
raise TypeError(f"Expected a dict or a list but got {type(obj)}: {obj}")
|
| 2034 |
+
|
| 2035 |
+
return cls.from_dict(from_yaml_inner(yaml_data))
|
| 2036 |
+
|
| 2037 |
+
def encode_example(self, example):
|
| 2038 |
+
"""
|
| 2039 |
+
Encode example into a format for Arrow.
|
| 2040 |
+
|
| 2041 |
+
Args:
|
| 2042 |
+
example (`dict[str, Any]`):
|
| 2043 |
+
Data in a Dataset row.
|
| 2044 |
+
|
| 2045 |
+
Returns:
|
| 2046 |
+
`dict[str, Any]`
|
| 2047 |
+
"""
|
| 2048 |
+
example = cast_to_python_objects(example)
|
| 2049 |
+
return encode_nested_example(self, example)
|
| 2050 |
+
|
| 2051 |
+
def encode_column(self, column, column_name: str):
|
| 2052 |
+
"""
|
| 2053 |
+
Encode column into a format for Arrow.
|
| 2054 |
+
|
| 2055 |
+
Args:
|
| 2056 |
+
column (`list[Any]`):
|
| 2057 |
+
Data in a Dataset column.
|
| 2058 |
+
column_name (`str`):
|
| 2059 |
+
Dataset column name.
|
| 2060 |
+
|
| 2061 |
+
Returns:
|
| 2062 |
+
`list[Any]`
|
| 2063 |
+
"""
|
| 2064 |
+
column = cast_to_python_objects(column)
|
| 2065 |
+
return [encode_nested_example(self[column_name], obj, level=1) for obj in column]
|
| 2066 |
+
|
| 2067 |
+
def encode_batch(self, batch):
|
| 2068 |
+
"""
|
| 2069 |
+
Encode batch into a format for Arrow.
|
| 2070 |
+
|
| 2071 |
+
Args:
|
| 2072 |
+
batch (`dict[str, list[Any]]`):
|
| 2073 |
+
Data in a Dataset batch.
|
| 2074 |
+
|
| 2075 |
+
Returns:
|
| 2076 |
+
`dict[str, list[Any]]`
|
| 2077 |
+
"""
|
| 2078 |
+
encoded_batch = {}
|
| 2079 |
+
if set(batch) != set(self):
|
| 2080 |
+
raise ValueError(f"Column mismatch between batch {set(batch)} and features {set(self)}")
|
| 2081 |
+
for key, column in batch.items():
|
| 2082 |
+
column = cast_to_python_objects(column)
|
| 2083 |
+
encoded_batch[key] = [encode_nested_example(self[key], obj, level=1) for obj in column]
|
| 2084 |
+
return encoded_batch
|
| 2085 |
+
|
| 2086 |
+
def decode_example(self, example: dict, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None):
|
| 2087 |
+
"""Decode example with custom feature decoding.
|
| 2088 |
+
|
| 2089 |
+
Args:
|
| 2090 |
+
example (`dict[str, Any]`):
|
| 2091 |
+
Dataset row data.
|
| 2092 |
+
token_per_repo_id (`dict`, *optional*):
|
| 2093 |
+
To access and decode audio or image files from private repositories on the Hub, you can pass
|
| 2094 |
+
a dictionary `repo_id (str) -> token (bool or str)`.
|
| 2095 |
+
|
| 2096 |
+
Returns:
|
| 2097 |
+
`dict[str, Any]`
|
| 2098 |
+
"""
|
| 2099 |
+
|
| 2100 |
+
return {
|
| 2101 |
+
column_name: decode_nested_example(feature, value, token_per_repo_id=token_per_repo_id)
|
| 2102 |
+
if self._column_requires_decoding[column_name]
|
| 2103 |
+
else value
|
| 2104 |
+
for column_name, (feature, value) in zip_dict(
|
| 2105 |
+
{key: value for key, value in self.items() if key in example}, example
|
| 2106 |
+
)
|
| 2107 |
+
}
|
| 2108 |
+
|
| 2109 |
+
def decode_column(self, column: list, column_name: str):
|
| 2110 |
+
"""Decode column with custom feature decoding.
|
| 2111 |
+
|
| 2112 |
+
Args:
|
| 2113 |
+
column (`list[Any]`):
|
| 2114 |
+
Dataset column data.
|
| 2115 |
+
column_name (`str`):
|
| 2116 |
+
Dataset column name.
|
| 2117 |
+
|
| 2118 |
+
Returns:
|
| 2119 |
+
`list[Any]`
|
| 2120 |
+
"""
|
| 2121 |
+
return (
|
| 2122 |
+
[decode_nested_example(self[column_name], value) if value is not None else None for value in column]
|
| 2123 |
+
if self._column_requires_decoding[column_name]
|
| 2124 |
+
else column
|
| 2125 |
+
)
|
| 2126 |
+
|
| 2127 |
+
def decode_batch(self, batch: dict, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None):
|
| 2128 |
+
"""Decode batch with custom feature decoding.
|
| 2129 |
+
|
| 2130 |
+
Args:
|
| 2131 |
+
batch (`dict[str, list[Any]]`):
|
| 2132 |
+
Dataset batch data.
|
| 2133 |
+
token_per_repo_id (`dict`, *optional*):
|
| 2134 |
+
To access and decode audio or image files from private repositories on the Hub, you can pass
|
| 2135 |
+
a dictionary repo_id (str) -> token (bool or str)
|
| 2136 |
+
|
| 2137 |
+
Returns:
|
| 2138 |
+
`dict[str, list[Any]]`
|
| 2139 |
+
"""
|
| 2140 |
+
decoded_batch = {}
|
| 2141 |
+
for column_name, column in batch.items():
|
| 2142 |
+
decoded_batch[column_name] = (
|
| 2143 |
+
[
|
| 2144 |
+
decode_nested_example(self[column_name], value, token_per_repo_id=token_per_repo_id)
|
| 2145 |
+
if value is not None
|
| 2146 |
+
else None
|
| 2147 |
+
for value in column
|
| 2148 |
+
]
|
| 2149 |
+
if self._column_requires_decoding[column_name]
|
| 2150 |
+
else column
|
| 2151 |
+
)
|
| 2152 |
+
return decoded_batch
|
| 2153 |
+
|
| 2154 |
+
def copy(self) -> "Features":
|
| 2155 |
+
"""
|
| 2156 |
+
Make a deep copy of [`Features`].
|
| 2157 |
+
|
| 2158 |
+
Returns:
|
| 2159 |
+
[`Features`]
|
| 2160 |
+
|
| 2161 |
+
Example:
|
| 2162 |
+
|
| 2163 |
+
```py
|
| 2164 |
+
>>> from datasets import load_dataset
|
| 2165 |
+
>>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
|
| 2166 |
+
>>> copy_of_features = ds.features.copy()
|
| 2167 |
+
>>> copy_of_features
|
| 2168 |
+
{'label': ClassLabel(names=['neg', 'pos'], id=None),
|
| 2169 |
+
'text': Value(dtype='string', id=None)}
|
| 2170 |
+
```
|
| 2171 |
+
"""
|
| 2172 |
+
return copy.deepcopy(self)
|
| 2173 |
+
|
| 2174 |
+
def reorder_fields_as(self, other: "Features") -> "Features":
|
| 2175 |
+
"""
|
| 2176 |
+
Reorder Features fields to match the field order of other [`Features`].
|
| 2177 |
+
|
| 2178 |
+
The order of the fields is important since it matters for the underlying arrow data.
|
| 2179 |
+
Re-ordering the fields allows to make the underlying arrow data type match.
|
| 2180 |
+
|
| 2181 |
+
Args:
|
| 2182 |
+
other ([`Features`]):
|
| 2183 |
+
The other [`Features`] to align with.
|
| 2184 |
+
|
| 2185 |
+
Returns:
|
| 2186 |
+
[`Features`]
|
| 2187 |
+
|
| 2188 |
+
Example::
|
| 2189 |
+
|
| 2190 |
+
>>> from datasets import Features, Sequence, Value
|
| 2191 |
+
>>> # let's say we have two features with a different order of nested fields (for a and b for example)
|
| 2192 |
+
>>> f1 = Features({"root": Sequence({"a": Value("string"), "b": Value("string")})})
|
| 2193 |
+
>>> f2 = Features({"root": {"b": Sequence(Value("string")), "a": Sequence(Value("string"))}})
|
| 2194 |
+
>>> assert f1.type != f2.type
|
| 2195 |
+
>>> # re-ordering keeps the base structure (here Sequence is defined at the root level), but makes the fields order match
|
| 2196 |
+
>>> f1.reorder_fields_as(f2)
|
| 2197 |
+
{'root': Sequence(feature={'b': Value(dtype='string', id=None), 'a': Value(dtype='string', id=None)}, length=-1, id=None)}
|
| 2198 |
+
>>> assert f1.reorder_fields_as(f2).type == f2.type
|
| 2199 |
+
"""
|
| 2200 |
+
|
| 2201 |
+
def recursive_reorder(source, target, stack=""):
|
| 2202 |
+
stack_position = " at " + stack[1:] if stack else ""
|
| 2203 |
+
if isinstance(target, Sequence):
|
| 2204 |
+
target = target.feature
|
| 2205 |
+
if isinstance(target, dict):
|
| 2206 |
+
target = {k: [v] for k, v in target.items()}
|
| 2207 |
+
else:
|
| 2208 |
+
target = [target]
|
| 2209 |
+
if isinstance(source, Sequence):
|
| 2210 |
+
sequence_kwargs = vars(source).copy()
|
| 2211 |
+
source = sequence_kwargs.pop("feature")
|
| 2212 |
+
if isinstance(source, dict):
|
| 2213 |
+
source = {k: [v] for k, v in source.items()}
|
| 2214 |
+
reordered = recursive_reorder(source, target, stack)
|
| 2215 |
+
return Sequence({k: v[0] for k, v in reordered.items()}, **sequence_kwargs)
|
| 2216 |
+
else:
|
| 2217 |
+
source = [source]
|
| 2218 |
+
reordered = recursive_reorder(source, target, stack)
|
| 2219 |
+
return Sequence(reordered[0], **sequence_kwargs)
|
| 2220 |
+
elif isinstance(source, dict):
|
| 2221 |
+
if not isinstance(target, dict):
|
| 2222 |
+
raise ValueError(f"Type mismatch: between {source} and {target}" + stack_position)
|
| 2223 |
+
if sorted(source) != sorted(target):
|
| 2224 |
+
message = (
|
| 2225 |
+
f"Keys mismatch: between {source} (source) and {target} (target).\n"
|
| 2226 |
+
f"{source.keys() - target.keys()} are missing from target "
|
| 2227 |
+
f"and {target.keys() - source.keys()} are missing from source" + stack_position
|
| 2228 |
+
)
|
| 2229 |
+
raise ValueError(message)
|
| 2230 |
+
return {key: recursive_reorder(source[key], target[key], stack + f".{key}") for key in target}
|
| 2231 |
+
elif isinstance(source, list):
|
| 2232 |
+
if not isinstance(target, list):
|
| 2233 |
+
raise ValueError(f"Type mismatch: between {source} and {target}" + stack_position)
|
| 2234 |
+
if len(source) != len(target):
|
| 2235 |
+
raise ValueError(f"Length mismatch: between {source} and {target}" + stack_position)
|
| 2236 |
+
return [recursive_reorder(source[i], target[i], stack + ".<list>") for i in range(len(target))]
|
| 2237 |
+
elif isinstance(source, LargeList):
|
| 2238 |
+
if not isinstance(target, LargeList):
|
| 2239 |
+
raise ValueError(f"Type mismatch: between {source} and {target}" + stack_position)
|
| 2240 |
+
return LargeList(recursive_reorder(source.feature, target.feature, stack))
|
| 2241 |
+
else:
|
| 2242 |
+
return source
|
| 2243 |
+
|
| 2244 |
+
return Features(recursive_reorder(self, other))
|
| 2245 |
+
|
| 2246 |
+
def flatten(self, max_depth=16) -> "Features":
|
| 2247 |
+
"""Flatten the features. Every dictionary column is removed and is replaced by
|
| 2248 |
+
all the subfields it contains. The new fields are named by concatenating the
|
| 2249 |
+
name of the original column and the subfield name like this: `<original>.<subfield>`.
|
| 2250 |
+
|
| 2251 |
+
If a column contains nested dictionaries, then all the lower-level subfields names are
|
| 2252 |
+
also concatenated to form new columns: `<original>.<subfield>.<subsubfield>`, etc.
|
| 2253 |
+
|
| 2254 |
+
Returns:
|
| 2255 |
+
[`Features`]:
|
| 2256 |
+
The flattened features.
|
| 2257 |
+
|
| 2258 |
+
Example:
|
| 2259 |
+
|
| 2260 |
+
```py
|
| 2261 |
+
>>> from datasets import load_dataset
|
| 2262 |
+
>>> ds = load_dataset("rajpurkar/squad", split="train")
|
| 2263 |
+
>>> ds.features.flatten()
|
| 2264 |
+
{'answers.answer_start': Sequence(feature=Value(dtype='int32', id=None), length=-1, id=None),
|
| 2265 |
+
'answers.text': Sequence(feature=Value(dtype='string', id=None), length=-1, id=None),
|
| 2266 |
+
'context': Value(dtype='string', id=None),
|
| 2267 |
+
'id': Value(dtype='string', id=None),
|
| 2268 |
+
'question': Value(dtype='string', id=None),
|
| 2269 |
+
'title': Value(dtype='string', id=None)}
|
| 2270 |
+
```
|
| 2271 |
+
"""
|
| 2272 |
+
for depth in range(1, max_depth):
|
| 2273 |
+
no_change = True
|
| 2274 |
+
flattened = self.copy()
|
| 2275 |
+
for column_name, subfeature in self.items():
|
| 2276 |
+
if isinstance(subfeature, dict):
|
| 2277 |
+
no_change = False
|
| 2278 |
+
flattened.update({f"{column_name}.{k}": v for k, v in subfeature.items()})
|
| 2279 |
+
del flattened[column_name]
|
| 2280 |
+
elif isinstance(subfeature, Sequence) and isinstance(subfeature.feature, dict):
|
| 2281 |
+
no_change = False
|
| 2282 |
+
flattened.update(
|
| 2283 |
+
{
|
| 2284 |
+
f"{column_name}.{k}": Sequence(v) if not isinstance(v, dict) else [v]
|
| 2285 |
+
for k, v in subfeature.feature.items()
|
| 2286 |
+
}
|
| 2287 |
+
)
|
| 2288 |
+
del flattened[column_name]
|
| 2289 |
+
elif hasattr(subfeature, "flatten") and subfeature.flatten() != subfeature:
|
| 2290 |
+
no_change = False
|
| 2291 |
+
flattened.update({f"{column_name}.{k}": v for k, v in subfeature.flatten().items()})
|
| 2292 |
+
del flattened[column_name]
|
| 2293 |
+
self = flattened
|
| 2294 |
+
if no_change:
|
| 2295 |
+
break
|
| 2296 |
+
return self
|
| 2297 |
+
|
| 2298 |
+
|
| 2299 |
+
def _align_features(features_list: list[Features]) -> list[Features]:
|
| 2300 |
+
"""Align dictionaries of features so that the keys that are found in multiple dictionaries share the same feature."""
|
| 2301 |
+
name2feature = {}
|
| 2302 |
+
for features in features_list:
|
| 2303 |
+
for k, v in features.items():
|
| 2304 |
+
if k in name2feature and isinstance(v, dict):
|
| 2305 |
+
# Recursively align features.
|
| 2306 |
+
name2feature[k] = _align_features([name2feature[k], v])[0]
|
| 2307 |
+
elif k not in name2feature or (isinstance(name2feature[k], Value) and name2feature[k].dtype == "null"):
|
| 2308 |
+
name2feature[k] = v
|
| 2309 |
+
|
| 2310 |
+
return [Features({k: name2feature[k] for k in features.keys()}) for features in features_list]
|
| 2311 |
+
|
| 2312 |
+
|
| 2313 |
+
def _check_if_features_can_be_aligned(features_list: list[Features]):
|
| 2314 |
+
"""Check if the dictionaries of features can be aligned.
|
| 2315 |
+
|
| 2316 |
+
Two dictonaries of features can be aligned if the keys they share have the same type or some of them is of type `Value("null")`.
|
| 2317 |
+
"""
|
| 2318 |
+
name2feature = {}
|
| 2319 |
+
for features in features_list:
|
| 2320 |
+
for k, v in features.items():
|
| 2321 |
+
if k not in name2feature or (isinstance(name2feature[k], Value) and name2feature[k].dtype == "null"):
|
| 2322 |
+
name2feature[k] = v
|
| 2323 |
+
|
| 2324 |
+
for features in features_list:
|
| 2325 |
+
for k, v in features.items():
|
| 2326 |
+
if isinstance(v, dict) and isinstance(name2feature[k], dict):
|
| 2327 |
+
# Deep checks for structure.
|
| 2328 |
+
_check_if_features_can_be_aligned([name2feature[k], v])
|
| 2329 |
+
elif not (isinstance(v, Value) and v.dtype == "null") and name2feature[k] != v:
|
| 2330 |
+
raise ValueError(
|
| 2331 |
+
f'The features can\'t be aligned because the key {k} of features {features} has unexpected type - {v} (expected either {name2feature[k]} or Value("null").'
|
| 2332 |
+
)
|
lib/python3.12/site-packages/datasets/features/image.py
ADDED
|
@@ -0,0 +1,382 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import sys
|
| 3 |
+
import warnings
|
| 4 |
+
from dataclasses import dataclass, field
|
| 5 |
+
from io import BytesIO
|
| 6 |
+
from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union
|
| 7 |
+
|
| 8 |
+
import numpy as np
|
| 9 |
+
import pyarrow as pa
|
| 10 |
+
|
| 11 |
+
from .. import config
|
| 12 |
+
from ..download.download_config import DownloadConfig
|
| 13 |
+
from ..table import array_cast
|
| 14 |
+
from ..utils.file_utils import is_local_path, xopen
|
| 15 |
+
from ..utils.py_utils import first_non_null_value, no_op_if_value_is_null, string_to_dict
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
if TYPE_CHECKING:
|
| 19 |
+
import PIL.Image
|
| 20 |
+
|
| 21 |
+
from .features import FeatureType
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
_IMAGE_COMPRESSION_FORMATS: Optional[list[str]] = None
|
| 25 |
+
_NATIVE_BYTEORDER = "<" if sys.byteorder == "little" else ">"
|
| 26 |
+
# Origin: https://github.com/python-pillow/Pillow/blob/698951e19e19972aeed56df686868f1329981c12/src/PIL/Image.py#L3126 minus "|i1" which values are not preserved correctly when saving and loading an image
|
| 27 |
+
_VALID_IMAGE_ARRAY_DTPYES = [
|
| 28 |
+
np.dtype("|b1"),
|
| 29 |
+
np.dtype("|u1"),
|
| 30 |
+
np.dtype("<u2"),
|
| 31 |
+
np.dtype(">u2"),
|
| 32 |
+
np.dtype("<i2"),
|
| 33 |
+
np.dtype(">i2"),
|
| 34 |
+
np.dtype("<u4"),
|
| 35 |
+
np.dtype(">u4"),
|
| 36 |
+
np.dtype("<i4"),
|
| 37 |
+
np.dtype(">i4"),
|
| 38 |
+
np.dtype("<f4"),
|
| 39 |
+
np.dtype(">f4"),
|
| 40 |
+
np.dtype("<f8"),
|
| 41 |
+
np.dtype(">f8"),
|
| 42 |
+
]
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
@dataclass
|
| 46 |
+
class Image:
|
| 47 |
+
"""Image [`Feature`] to read image data from an image file.
|
| 48 |
+
|
| 49 |
+
Input: The Image feature accepts as input:
|
| 50 |
+
- A `str`: Absolute path to the image file (i.e. random access is allowed).
|
| 51 |
+
- A `dict` with the keys:
|
| 52 |
+
|
| 53 |
+
- `path`: String with relative path of the image file to the archive file.
|
| 54 |
+
- `bytes`: Bytes of the image file.
|
| 55 |
+
|
| 56 |
+
This is useful for archived files with sequential access.
|
| 57 |
+
|
| 58 |
+
- An `np.ndarray`: NumPy array representing an image.
|
| 59 |
+
- A `PIL.Image.Image`: PIL image object.
|
| 60 |
+
|
| 61 |
+
Args:
|
| 62 |
+
mode (`str`, *optional*):
|
| 63 |
+
The mode to convert the image to. If `None`, the native mode of the image is used.
|
| 64 |
+
decode (`bool`, defaults to `True`):
|
| 65 |
+
Whether to decode the image data. If `False`,
|
| 66 |
+
returns the underlying dictionary in the format `{"path": image_path, "bytes": image_bytes}`.
|
| 67 |
+
|
| 68 |
+
Examples:
|
| 69 |
+
|
| 70 |
+
```py
|
| 71 |
+
>>> from datasets import load_dataset, Image
|
| 72 |
+
>>> ds = load_dataset("AI-Lab-Makerere/beans", split="train")
|
| 73 |
+
>>> ds.features["image"]
|
| 74 |
+
Image(decode=True, id=None)
|
| 75 |
+
>>> ds[0]["image"]
|
| 76 |
+
<PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=500x500 at 0x15E52E7F0>
|
| 77 |
+
>>> ds = ds.cast_column('image', Image(decode=False))
|
| 78 |
+
{'bytes': None,
|
| 79 |
+
'path': '/root/.cache/huggingface/datasets/downloads/extracted/b0a21163f78769a2cf11f58dfc767fb458fc7cea5c05dccc0144a2c0f0bc1292/train/healthy/healthy_train.85.jpg'}
|
| 80 |
+
```
|
| 81 |
+
"""
|
| 82 |
+
|
| 83 |
+
mode: Optional[str] = None
|
| 84 |
+
decode: bool = True
|
| 85 |
+
id: Optional[str] = None
|
| 86 |
+
# Automatically constructed
|
| 87 |
+
dtype: ClassVar[str] = "PIL.Image.Image"
|
| 88 |
+
pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()})
|
| 89 |
+
_type: str = field(default="Image", init=False, repr=False)
|
| 90 |
+
|
| 91 |
+
def __call__(self):
|
| 92 |
+
return self.pa_type
|
| 93 |
+
|
| 94 |
+
def encode_example(self, value: Union[str, bytes, bytearray, dict, np.ndarray, "PIL.Image.Image"]) -> dict:
|
| 95 |
+
"""Encode example into a format for Arrow.
|
| 96 |
+
|
| 97 |
+
Args:
|
| 98 |
+
value (`str`, `np.ndarray`, `PIL.Image.Image` or `dict`):
|
| 99 |
+
Data passed as input to Image feature.
|
| 100 |
+
|
| 101 |
+
Returns:
|
| 102 |
+
`dict` with "path" and "bytes" fields
|
| 103 |
+
"""
|
| 104 |
+
if config.PIL_AVAILABLE:
|
| 105 |
+
import PIL.Image
|
| 106 |
+
else:
|
| 107 |
+
raise ImportError("To support encoding images, please install 'Pillow'.")
|
| 108 |
+
|
| 109 |
+
if isinstance(value, list):
|
| 110 |
+
value = np.array(value)
|
| 111 |
+
|
| 112 |
+
if isinstance(value, str):
|
| 113 |
+
return {"path": value, "bytes": None}
|
| 114 |
+
elif isinstance(value, (bytes, bytearray)):
|
| 115 |
+
return {"path": None, "bytes": value}
|
| 116 |
+
elif isinstance(value, np.ndarray):
|
| 117 |
+
# convert the image array to PNG/TIFF bytes
|
| 118 |
+
return encode_np_array(value)
|
| 119 |
+
elif isinstance(value, PIL.Image.Image):
|
| 120 |
+
# convert the PIL image to bytes (default format is PNG/TIFF)
|
| 121 |
+
return encode_pil_image(value)
|
| 122 |
+
elif value.get("path") is not None and os.path.isfile(value["path"]):
|
| 123 |
+
# we set "bytes": None to not duplicate the data if they're already available locally
|
| 124 |
+
return {"bytes": None, "path": value.get("path")}
|
| 125 |
+
elif value.get("bytes") is not None or value.get("path") is not None:
|
| 126 |
+
# store the image bytes, and path is used to infer the image format using the file extension
|
| 127 |
+
return {"bytes": value.get("bytes"), "path": value.get("path")}
|
| 128 |
+
else:
|
| 129 |
+
raise ValueError(
|
| 130 |
+
f"An image sample should have one of 'path' or 'bytes' but they are missing or None in {value}."
|
| 131 |
+
)
|
| 132 |
+
|
| 133 |
+
def decode_example(self, value: dict, token_per_repo_id=None) -> "PIL.Image.Image":
|
| 134 |
+
"""Decode example image file into image data.
|
| 135 |
+
|
| 136 |
+
Args:
|
| 137 |
+
value (`str` or `dict`):
|
| 138 |
+
A string with the absolute image file path, a dictionary with
|
| 139 |
+
keys:
|
| 140 |
+
|
| 141 |
+
- `path`: String with absolute or relative image file path.
|
| 142 |
+
- `bytes`: The bytes of the image file.
|
| 143 |
+
token_per_repo_id (`dict`, *optional*):
|
| 144 |
+
To access and decode
|
| 145 |
+
image files from private repositories on the Hub, you can pass
|
| 146 |
+
a dictionary repo_id (`str`) -> token (`bool` or `str`).
|
| 147 |
+
|
| 148 |
+
Returns:
|
| 149 |
+
`PIL.Image.Image`
|
| 150 |
+
"""
|
| 151 |
+
if not self.decode:
|
| 152 |
+
raise RuntimeError("Decoding is disabled for this feature. Please use Image(decode=True) instead.")
|
| 153 |
+
|
| 154 |
+
if config.PIL_AVAILABLE:
|
| 155 |
+
import PIL.Image
|
| 156 |
+
import PIL.ImageOps
|
| 157 |
+
else:
|
| 158 |
+
raise ImportError("To support decoding images, please install 'Pillow'.")
|
| 159 |
+
|
| 160 |
+
if token_per_repo_id is None:
|
| 161 |
+
token_per_repo_id = {}
|
| 162 |
+
|
| 163 |
+
path, bytes_ = value["path"], value["bytes"]
|
| 164 |
+
if bytes_ is None:
|
| 165 |
+
if path is None:
|
| 166 |
+
raise ValueError(f"An image should have one of 'path' or 'bytes' but both are None in {value}.")
|
| 167 |
+
else:
|
| 168 |
+
if is_local_path(path):
|
| 169 |
+
image = PIL.Image.open(path)
|
| 170 |
+
else:
|
| 171 |
+
source_url = path.split("::")[-1]
|
| 172 |
+
pattern = (
|
| 173 |
+
config.HUB_DATASETS_URL
|
| 174 |
+
if source_url.startswith(config.HF_ENDPOINT)
|
| 175 |
+
else config.HUB_DATASETS_HFFS_URL
|
| 176 |
+
)
|
| 177 |
+
source_url_fields = string_to_dict(source_url, pattern)
|
| 178 |
+
token = (
|
| 179 |
+
token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None
|
| 180 |
+
)
|
| 181 |
+
download_config = DownloadConfig(token=token)
|
| 182 |
+
with xopen(path, "rb", download_config=download_config) as f:
|
| 183 |
+
bytes_ = BytesIO(f.read())
|
| 184 |
+
image = PIL.Image.open(bytes_)
|
| 185 |
+
else:
|
| 186 |
+
image = PIL.Image.open(BytesIO(bytes_))
|
| 187 |
+
image.load() # to avoid "Too many open files" errors
|
| 188 |
+
if image.getexif().get(PIL.Image.ExifTags.Base.Orientation) is not None:
|
| 189 |
+
image = PIL.ImageOps.exif_transpose(image)
|
| 190 |
+
if self.mode and self.mode != image.mode:
|
| 191 |
+
image = image.convert(self.mode)
|
| 192 |
+
return image
|
| 193 |
+
|
| 194 |
+
def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]:
|
| 195 |
+
"""If in the decodable state, return the feature itself, otherwise flatten the feature into a dictionary."""
|
| 196 |
+
from .features import Value
|
| 197 |
+
|
| 198 |
+
return (
|
| 199 |
+
self
|
| 200 |
+
if self.decode
|
| 201 |
+
else {
|
| 202 |
+
"bytes": Value("binary"),
|
| 203 |
+
"path": Value("string"),
|
| 204 |
+
}
|
| 205 |
+
)
|
| 206 |
+
|
| 207 |
+
def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray, pa.ListArray]) -> pa.StructArray:
|
| 208 |
+
"""Cast an Arrow array to the Image arrow storage type.
|
| 209 |
+
The Arrow types that can be converted to the Image pyarrow storage type are:
|
| 210 |
+
|
| 211 |
+
- `pa.string()` - it must contain the "path" data
|
| 212 |
+
- `pa.binary()` - it must contain the image bytes
|
| 213 |
+
- `pa.struct({"bytes": pa.binary()})`
|
| 214 |
+
- `pa.struct({"path": pa.string()})`
|
| 215 |
+
- `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter
|
| 216 |
+
- `pa.list(*)` - it must contain the image array data
|
| 217 |
+
|
| 218 |
+
Args:
|
| 219 |
+
storage (`Union[pa.StringArray, pa.StructArray, pa.ListArray]`):
|
| 220 |
+
PyArrow array to cast.
|
| 221 |
+
|
| 222 |
+
Returns:
|
| 223 |
+
`pa.StructArray`: Array in the Image arrow storage type, that is
|
| 224 |
+
`pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
|
| 225 |
+
"""
|
| 226 |
+
if pa.types.is_string(storage.type):
|
| 227 |
+
bytes_array = pa.array([None] * len(storage), type=pa.binary())
|
| 228 |
+
storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null())
|
| 229 |
+
elif pa.types.is_binary(storage.type):
|
| 230 |
+
path_array = pa.array([None] * len(storage), type=pa.string())
|
| 231 |
+
storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null())
|
| 232 |
+
elif pa.types.is_struct(storage.type):
|
| 233 |
+
if storage.type.get_field_index("bytes") >= 0:
|
| 234 |
+
bytes_array = storage.field("bytes")
|
| 235 |
+
else:
|
| 236 |
+
bytes_array = pa.array([None] * len(storage), type=pa.binary())
|
| 237 |
+
if storage.type.get_field_index("path") >= 0:
|
| 238 |
+
path_array = storage.field("path")
|
| 239 |
+
else:
|
| 240 |
+
path_array = pa.array([None] * len(storage), type=pa.string())
|
| 241 |
+
storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null())
|
| 242 |
+
elif pa.types.is_list(storage.type):
|
| 243 |
+
bytes_array = pa.array(
|
| 244 |
+
[encode_np_array(np.array(arr))["bytes"] if arr is not None else None for arr in storage.to_pylist()],
|
| 245 |
+
type=pa.binary(),
|
| 246 |
+
)
|
| 247 |
+
path_array = pa.array([None] * len(storage), type=pa.string())
|
| 248 |
+
storage = pa.StructArray.from_arrays(
|
| 249 |
+
[bytes_array, path_array], ["bytes", "path"], mask=bytes_array.is_null()
|
| 250 |
+
)
|
| 251 |
+
return array_cast(storage, self.pa_type)
|
| 252 |
+
|
| 253 |
+
def embed_storage(self, storage: pa.StructArray) -> pa.StructArray:
|
| 254 |
+
"""Embed image files into the Arrow array.
|
| 255 |
+
|
| 256 |
+
Args:
|
| 257 |
+
storage (`pa.StructArray`):
|
| 258 |
+
PyArrow array to embed.
|
| 259 |
+
|
| 260 |
+
Returns:
|
| 261 |
+
`pa.StructArray`: Array in the Image arrow storage type, that is
|
| 262 |
+
`pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
|
| 263 |
+
"""
|
| 264 |
+
|
| 265 |
+
@no_op_if_value_is_null
|
| 266 |
+
def path_to_bytes(path):
|
| 267 |
+
with xopen(path, "rb") as f:
|
| 268 |
+
bytes_ = f.read()
|
| 269 |
+
return bytes_
|
| 270 |
+
|
| 271 |
+
bytes_array = pa.array(
|
| 272 |
+
[
|
| 273 |
+
(path_to_bytes(x["path"]) if x["bytes"] is None else x["bytes"]) if x is not None else None
|
| 274 |
+
for x in storage.to_pylist()
|
| 275 |
+
],
|
| 276 |
+
type=pa.binary(),
|
| 277 |
+
)
|
| 278 |
+
path_array = pa.array(
|
| 279 |
+
[os.path.basename(path) if path is not None else None for path in storage.field("path").to_pylist()],
|
| 280 |
+
type=pa.string(),
|
| 281 |
+
)
|
| 282 |
+
storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=bytes_array.is_null())
|
| 283 |
+
return array_cast(storage, self.pa_type)
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
def list_image_compression_formats() -> list[str]:
|
| 287 |
+
if config.PIL_AVAILABLE:
|
| 288 |
+
import PIL.Image
|
| 289 |
+
else:
|
| 290 |
+
raise ImportError("To support encoding images, please install 'Pillow'.")
|
| 291 |
+
|
| 292 |
+
global _IMAGE_COMPRESSION_FORMATS
|
| 293 |
+
if _IMAGE_COMPRESSION_FORMATS is None:
|
| 294 |
+
PIL.Image.init()
|
| 295 |
+
_IMAGE_COMPRESSION_FORMATS = list(set(PIL.Image.OPEN.keys()) & set(PIL.Image.SAVE.keys()))
|
| 296 |
+
return _IMAGE_COMPRESSION_FORMATS
|
| 297 |
+
|
| 298 |
+
|
| 299 |
+
def image_to_bytes(image: "PIL.Image.Image") -> bytes:
|
| 300 |
+
"""Convert a PIL Image object to bytes using native compression if possible, otherwise use PNG/TIFF compression."""
|
| 301 |
+
buffer = BytesIO()
|
| 302 |
+
if image.format in list_image_compression_formats():
|
| 303 |
+
format = image.format
|
| 304 |
+
else:
|
| 305 |
+
format = "PNG" if image.mode in ["1", "L", "LA", "RGB", "RGBA"] else "TIFF"
|
| 306 |
+
image.save(buffer, format=format)
|
| 307 |
+
return buffer.getvalue()
|
| 308 |
+
|
| 309 |
+
|
| 310 |
+
def encode_pil_image(image: "PIL.Image.Image") -> dict:
|
| 311 |
+
if hasattr(image, "filename") and image.filename != "":
|
| 312 |
+
return {"path": image.filename, "bytes": None}
|
| 313 |
+
else:
|
| 314 |
+
return {"path": None, "bytes": image_to_bytes(image)}
|
| 315 |
+
|
| 316 |
+
|
| 317 |
+
def encode_np_array(array: np.ndarray) -> dict:
|
| 318 |
+
if config.PIL_AVAILABLE:
|
| 319 |
+
import PIL.Image
|
| 320 |
+
else:
|
| 321 |
+
raise ImportError("To support encoding images, please install 'Pillow'.")
|
| 322 |
+
|
| 323 |
+
dtype = array.dtype
|
| 324 |
+
dtype_byteorder = dtype.byteorder if dtype.byteorder != "=" else _NATIVE_BYTEORDER
|
| 325 |
+
dtype_kind = dtype.kind
|
| 326 |
+
dtype_itemsize = dtype.itemsize
|
| 327 |
+
|
| 328 |
+
dest_dtype = None
|
| 329 |
+
|
| 330 |
+
# Multi-channel array case (only np.dtype("|u1") is allowed)
|
| 331 |
+
if array.shape[2:]:
|
| 332 |
+
if dtype_kind not in ["u", "i"]:
|
| 333 |
+
raise TypeError(
|
| 334 |
+
f"Unsupported array dtype {dtype} for image encoding. Only {dest_dtype} is supported for multi-channel arrays."
|
| 335 |
+
)
|
| 336 |
+
dest_dtype = np.dtype("|u1")
|
| 337 |
+
if dtype != dest_dtype:
|
| 338 |
+
warnings.warn(f"Downcasting array dtype {dtype} to {dest_dtype} to be compatible with 'Pillow'")
|
| 339 |
+
# Exact match
|
| 340 |
+
elif dtype in _VALID_IMAGE_ARRAY_DTPYES:
|
| 341 |
+
dest_dtype = dtype
|
| 342 |
+
else: # Downcast the type within the kind (np.can_cast(from_type, to_type, casting="same_kind") doesn't behave as expected, so do it manually)
|
| 343 |
+
while dtype_itemsize >= 1:
|
| 344 |
+
dtype_str = dtype_byteorder + dtype_kind + str(dtype_itemsize)
|
| 345 |
+
if np.dtype(dtype_str) in _VALID_IMAGE_ARRAY_DTPYES:
|
| 346 |
+
dest_dtype = np.dtype(dtype_str)
|
| 347 |
+
warnings.warn(f"Downcasting array dtype {dtype} to {dest_dtype} to be compatible with 'Pillow'")
|
| 348 |
+
break
|
| 349 |
+
else:
|
| 350 |
+
dtype_itemsize //= 2
|
| 351 |
+
if dest_dtype is None:
|
| 352 |
+
raise TypeError(
|
| 353 |
+
f"Cannot downcast dtype {dtype} to a valid image dtype. Valid image dtypes: {_VALID_IMAGE_ARRAY_DTPYES}"
|
| 354 |
+
)
|
| 355 |
+
|
| 356 |
+
image = PIL.Image.fromarray(array.astype(dest_dtype))
|
| 357 |
+
return {"path": None, "bytes": image_to_bytes(image)}
|
| 358 |
+
|
| 359 |
+
|
| 360 |
+
def objects_to_list_of_image_dicts(
|
| 361 |
+
objs: Union[list[str], list[dict], list[np.ndarray], list["PIL.Image.Image"]],
|
| 362 |
+
) -> list[dict]:
|
| 363 |
+
"""Encode a list of objects into a format suitable for creating an extension array of type `ImageExtensionType`."""
|
| 364 |
+
if config.PIL_AVAILABLE:
|
| 365 |
+
import PIL.Image
|
| 366 |
+
else:
|
| 367 |
+
raise ImportError("To support encoding images, please install 'Pillow'.")
|
| 368 |
+
|
| 369 |
+
if objs:
|
| 370 |
+
_, obj = first_non_null_value(objs)
|
| 371 |
+
if isinstance(obj, str):
|
| 372 |
+
return [{"path": obj, "bytes": None} if obj is not None else None for obj in objs]
|
| 373 |
+
if isinstance(obj, np.ndarray):
|
| 374 |
+
obj_to_image_dict_func = no_op_if_value_is_null(encode_np_array)
|
| 375 |
+
return [obj_to_image_dict_func(obj) for obj in objs]
|
| 376 |
+
elif isinstance(obj, PIL.Image.Image):
|
| 377 |
+
obj_to_image_dict_func = no_op_if_value_is_null(encode_pil_image)
|
| 378 |
+
return [obj_to_image_dict_func(obj) for obj in objs]
|
| 379 |
+
else:
|
| 380 |
+
return objs
|
| 381 |
+
else:
|
| 382 |
+
return objs
|
lib/python3.12/site-packages/datasets/features/pdf.py
ADDED
|
@@ -0,0 +1,238 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from dataclasses import dataclass, field
|
| 3 |
+
from io import BytesIO
|
| 4 |
+
from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union
|
| 5 |
+
|
| 6 |
+
import pyarrow as pa
|
| 7 |
+
|
| 8 |
+
from .. import config
|
| 9 |
+
from ..download.download_config import DownloadConfig
|
| 10 |
+
from ..table import array_cast
|
| 11 |
+
from ..utils.file_utils import is_local_path, xopen
|
| 12 |
+
from ..utils.py_utils import string_to_dict
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
if TYPE_CHECKING:
|
| 16 |
+
import pdfplumber
|
| 17 |
+
|
| 18 |
+
from .features import FeatureType
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def pdf_to_bytes(pdf: "pdfplumber.pdf.PDF") -> bytes:
|
| 22 |
+
"""Convert a pdfplumber.pdf.PDF object to bytes."""
|
| 23 |
+
with BytesIO() as buffer:
|
| 24 |
+
for page in pdf.pages:
|
| 25 |
+
buffer.write(page.pdf.stream)
|
| 26 |
+
return buffer.getvalue()
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
@dataclass
|
| 30 |
+
class Pdf:
|
| 31 |
+
"""
|
| 32 |
+
**Experimental.**
|
| 33 |
+
Pdf [`Feature`] to read pdf documents from a pdf file.
|
| 34 |
+
|
| 35 |
+
Input: The Pdf feature accepts as input:
|
| 36 |
+
- A `str`: Absolute path to the pdf file (i.e. random access is allowed).
|
| 37 |
+
- A `dict` with the keys:
|
| 38 |
+
- `path`: String with relative path of the pdf file in a dataset repository.
|
| 39 |
+
- `bytes`: Bytes of the pdf file.
|
| 40 |
+
This is useful for archived files with sequential access.
|
| 41 |
+
|
| 42 |
+
- A `pdfplumber.pdf.PDF`: pdfplumber pdf object.
|
| 43 |
+
|
| 44 |
+
Args:
|
| 45 |
+
mode (`str`, *optional*):
|
| 46 |
+
The mode to convert the pdf to. If `None`, the native mode of the pdf is used.
|
| 47 |
+
decode (`bool`, defaults to `True`):
|
| 48 |
+
Whether to decode the pdf data. If `False`,
|
| 49 |
+
returns the underlying dictionary in the format `{"path": pdf_path, "bytes": pdf_bytes}`.
|
| 50 |
+
|
| 51 |
+
Examples:
|
| 52 |
+
|
| 53 |
+
```py
|
| 54 |
+
>>> from datasets import Dataset, Pdf
|
| 55 |
+
>>> ds = Dataset.from_dict({"pdf": ["path/to/pdf/file.pdf"]}).cast_column("pdf", Pdf())
|
| 56 |
+
>>> ds.features["pdf"]
|
| 57 |
+
Pdf(decode=True, id=None)
|
| 58 |
+
>>> ds[0]["pdf"]
|
| 59 |
+
<pdfplumber.pdf.PDF object at 0x7f8a1c2d8f40>
|
| 60 |
+
>>> ds = ds.cast_column("pdf", Pdf(decode=False))
|
| 61 |
+
>>> ds[0]["pdf"]
|
| 62 |
+
{'bytes': None,
|
| 63 |
+
'path': 'path/to/pdf/file.pdf'}
|
| 64 |
+
```
|
| 65 |
+
"""
|
| 66 |
+
|
| 67 |
+
decode: bool = True
|
| 68 |
+
id: Optional[str] = None
|
| 69 |
+
|
| 70 |
+
# Automatically constructed
|
| 71 |
+
dtype: ClassVar[str] = "pdfplumber.pdf.PDF"
|
| 72 |
+
pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()})
|
| 73 |
+
_type: str = field(default="Pdf", init=False, repr=False)
|
| 74 |
+
|
| 75 |
+
def __call__(self):
|
| 76 |
+
return self.pa_type
|
| 77 |
+
|
| 78 |
+
def encode_example(self, value: Union[str, bytes, bytearray, dict, "pdfplumber.pdf.PDF"]) -> dict:
|
| 79 |
+
"""Encode example into a format for Arrow.
|
| 80 |
+
|
| 81 |
+
Args:
|
| 82 |
+
value (`str`, `bytes`, `pdfplumber.pdf.PDF` or `dict`):
|
| 83 |
+
Data passed as input to Pdf feature.
|
| 84 |
+
|
| 85 |
+
Returns:
|
| 86 |
+
`dict` with "path" and "bytes" fields
|
| 87 |
+
"""
|
| 88 |
+
if config.PDFPLUMBER_AVAILABLE:
|
| 89 |
+
import pdfplumber
|
| 90 |
+
else:
|
| 91 |
+
pdfplumber = None
|
| 92 |
+
|
| 93 |
+
if isinstance(value, str):
|
| 94 |
+
return {"path": value, "bytes": None}
|
| 95 |
+
elif isinstance(value, (bytes, bytearray)):
|
| 96 |
+
return {"path": None, "bytes": value}
|
| 97 |
+
elif pdfplumber is not None and isinstance(value, pdfplumber.pdf.PDF):
|
| 98 |
+
# convert the pdfplumber.pdf.PDF to bytes
|
| 99 |
+
return encode_pdfplumber_pdf(value)
|
| 100 |
+
elif value.get("path") is not None and os.path.isfile(value["path"]):
|
| 101 |
+
# we set "bytes": None to not duplicate the data if they're already available locally
|
| 102 |
+
return {"bytes": None, "path": value.get("path")}
|
| 103 |
+
elif value.get("bytes") is not None or value.get("path") is not None:
|
| 104 |
+
# store the pdf bytes, and path is used to infer the pdf format using the file extension
|
| 105 |
+
return {"bytes": value.get("bytes"), "path": value.get("path")}
|
| 106 |
+
else:
|
| 107 |
+
raise ValueError(
|
| 108 |
+
f"A pdf sample should have one of 'path' or 'bytes' but they are missing or None in {value}."
|
| 109 |
+
)
|
| 110 |
+
|
| 111 |
+
def decode_example(self, value: dict, token_per_repo_id=None) -> "pdfplumber.pdf.PDF":
|
| 112 |
+
"""Decode example pdf file into pdf data.
|
| 113 |
+
|
| 114 |
+
Args:
|
| 115 |
+
value (`str` or `dict`):
|
| 116 |
+
A string with the absolute pdf file path, a dictionary with
|
| 117 |
+
keys:
|
| 118 |
+
|
| 119 |
+
- `path`: String with absolute or relative pdf file path.
|
| 120 |
+
- `bytes`: The bytes of the pdf file.
|
| 121 |
+
|
| 122 |
+
token_per_repo_id (`dict`, *optional*):
|
| 123 |
+
To access and decode pdf files from private repositories on
|
| 124 |
+
the Hub, you can pass a dictionary
|
| 125 |
+
repo_id (`str`) -> token (`bool` or `str`).
|
| 126 |
+
|
| 127 |
+
Returns:
|
| 128 |
+
`pdfplumber.pdf.PDF`
|
| 129 |
+
"""
|
| 130 |
+
if not self.decode:
|
| 131 |
+
raise RuntimeError("Decoding is disabled for this feature. Please use Pdf(decode=True) instead.")
|
| 132 |
+
|
| 133 |
+
if config.PDFPLUMBER_AVAILABLE:
|
| 134 |
+
import pdfplumber
|
| 135 |
+
else:
|
| 136 |
+
raise ImportError("To support decoding pdfs, please install 'pdfplumber'.")
|
| 137 |
+
|
| 138 |
+
if token_per_repo_id is None:
|
| 139 |
+
token_per_repo_id = {}
|
| 140 |
+
|
| 141 |
+
path, bytes_ = value["path"], value["bytes"]
|
| 142 |
+
if bytes_ is None:
|
| 143 |
+
if path is None:
|
| 144 |
+
raise ValueError(f"A pdf should have one of 'path' or 'bytes' but both are None in {value}.")
|
| 145 |
+
else:
|
| 146 |
+
if is_local_path(path):
|
| 147 |
+
pdf = pdfplumber.open(path)
|
| 148 |
+
else:
|
| 149 |
+
source_url = path.split("::")[-1]
|
| 150 |
+
pattern = (
|
| 151 |
+
config.HUB_DATASETS_URL
|
| 152 |
+
if source_url.startswith(config.HF_ENDPOINT)
|
| 153 |
+
else config.HUB_DATASETS_HFFS_URL
|
| 154 |
+
)
|
| 155 |
+
try:
|
| 156 |
+
repo_id = string_to_dict(source_url, pattern)["repo_id"]
|
| 157 |
+
token = token_per_repo_id.get(repo_id)
|
| 158 |
+
except ValueError:
|
| 159 |
+
token = None
|
| 160 |
+
download_config = DownloadConfig(token=token)
|
| 161 |
+
f = xopen(path, "rb", download_config=download_config)
|
| 162 |
+
return pdfplumber.open(f)
|
| 163 |
+
else:
|
| 164 |
+
with pdfplumber.open(BytesIO(bytes_)) as p:
|
| 165 |
+
pdf = p
|
| 166 |
+
|
| 167 |
+
return pdf
|
| 168 |
+
|
| 169 |
+
def flatten(self) -> Union["FeatureType", Dict[str, "FeatureType"]]:
|
| 170 |
+
"""If in the decodable state, return the feature itself, otherwise flatten the feature into a dictionary."""
|
| 171 |
+
from .features import Value
|
| 172 |
+
|
| 173 |
+
return (
|
| 174 |
+
self
|
| 175 |
+
if self.decode
|
| 176 |
+
else {
|
| 177 |
+
"bytes": Value("binary"),
|
| 178 |
+
"path": Value("string"),
|
| 179 |
+
}
|
| 180 |
+
)
|
| 181 |
+
|
| 182 |
+
def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray, pa.ListArray]) -> pa.StructArray:
|
| 183 |
+
"""Cast an Arrow array to the Pdf arrow storage type.
|
| 184 |
+
The Arrow types that can be converted to the Pdf pyarrow storage type are:
|
| 185 |
+
|
| 186 |
+
- `pa.string()` - it must contain the "path" data
|
| 187 |
+
- `pa.binary()` - it must contain the image bytes
|
| 188 |
+
- `pa.struct({"bytes": pa.binary()})`
|
| 189 |
+
- `pa.struct({"path": pa.string()})`
|
| 190 |
+
- `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter
|
| 191 |
+
- `pa.list(*)` - it must contain the pdf array data
|
| 192 |
+
|
| 193 |
+
Args:
|
| 194 |
+
storage (`Union[pa.StringArray, pa.StructArray, pa.ListArray]`):
|
| 195 |
+
PyArrow array to cast.
|
| 196 |
+
|
| 197 |
+
Returns:
|
| 198 |
+
`pa.StructArray`: Array in the Pdf arrow storage type, that is
|
| 199 |
+
`pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
|
| 200 |
+
"""
|
| 201 |
+
if pa.types.is_string(storage.type):
|
| 202 |
+
bytes_array = pa.array([None] * len(storage), type=pa.binary())
|
| 203 |
+
storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null())
|
| 204 |
+
elif pa.types.is_binary(storage.type):
|
| 205 |
+
path_array = pa.array([None] * len(storage), type=pa.string())
|
| 206 |
+
storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null())
|
| 207 |
+
elif pa.types.is_struct(storage.type):
|
| 208 |
+
if storage.type.get_field_index("bytes") >= 0:
|
| 209 |
+
bytes_array = storage.field("bytes")
|
| 210 |
+
else:
|
| 211 |
+
bytes_array = pa.array([None] * len(storage), type=pa.binary())
|
| 212 |
+
if storage.type.get_field_index("path") >= 0:
|
| 213 |
+
path_array = storage.field("path")
|
| 214 |
+
else:
|
| 215 |
+
path_array = pa.array([None] * len(storage), type=pa.string())
|
| 216 |
+
storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null())
|
| 217 |
+
return array_cast(storage, self.pa_type)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
def encode_pdfplumber_pdf(pdf: "pdfplumber.pdf.PDF") -> dict:
|
| 221 |
+
"""
|
| 222 |
+
Encode a pdfplumber.pdf.PDF object into a dictionary.
|
| 223 |
+
|
| 224 |
+
If the PDF has an associated file path, returns the path. Otherwise, serializes
|
| 225 |
+
the PDF content into bytes.
|
| 226 |
+
|
| 227 |
+
Args:
|
| 228 |
+
pdf (pdfplumber.pdf.PDF): A pdfplumber PDF object.
|
| 229 |
+
|
| 230 |
+
Returns:
|
| 231 |
+
dict: A dictionary with "path" or "bytes" field.
|
| 232 |
+
"""
|
| 233 |
+
if hasattr(pdf, "stream") and hasattr(pdf.stream, "name") and pdf.stream.name:
|
| 234 |
+
# Return the path if the PDF has an associated file path
|
| 235 |
+
return {"path": pdf.stream.name, "bytes": None}
|
| 236 |
+
else:
|
| 237 |
+
# Convert the PDF to bytes if no path is available
|
| 238 |
+
return {"path": None, "bytes": pdf_to_bytes(pdf)}
|
lib/python3.12/site-packages/datasets/features/translation.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass, field
|
| 2 |
+
from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union
|
| 3 |
+
|
| 4 |
+
import pyarrow as pa
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
if TYPE_CHECKING:
|
| 8 |
+
from .features import FeatureType
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
@dataclass
|
| 12 |
+
class Translation:
|
| 13 |
+
"""`Feature` for translations with fixed languages per example.
|
| 14 |
+
Here for compatibility with tfds.
|
| 15 |
+
|
| 16 |
+
Args:
|
| 17 |
+
languages (`dict`):
|
| 18 |
+
A dictionary for each example mapping string language codes to string translations.
|
| 19 |
+
|
| 20 |
+
Example:
|
| 21 |
+
|
| 22 |
+
```python
|
| 23 |
+
>>> # At construction time:
|
| 24 |
+
>>> datasets.features.Translation(languages=['en', 'fr', 'de'])
|
| 25 |
+
>>> # During data generation:
|
| 26 |
+
>>> yield {
|
| 27 |
+
... 'en': 'the cat',
|
| 28 |
+
... 'fr': 'le chat',
|
| 29 |
+
... 'de': 'die katze'
|
| 30 |
+
... }
|
| 31 |
+
```
|
| 32 |
+
"""
|
| 33 |
+
|
| 34 |
+
languages: list[str]
|
| 35 |
+
id: Optional[str] = None
|
| 36 |
+
# Automatically constructed
|
| 37 |
+
dtype: ClassVar[str] = "dict"
|
| 38 |
+
pa_type: ClassVar[Any] = None
|
| 39 |
+
_type: str = field(default="Translation", init=False, repr=False)
|
| 40 |
+
|
| 41 |
+
def __call__(self):
|
| 42 |
+
return pa.struct({lang: pa.string() for lang in sorted(self.languages)})
|
| 43 |
+
|
| 44 |
+
def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]:
|
| 45 |
+
"""Flatten the Translation feature into a dictionary."""
|
| 46 |
+
from .features import Value
|
| 47 |
+
|
| 48 |
+
return {k: Value("string") for k in sorted(self.languages)}
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
@dataclass
|
| 52 |
+
class TranslationVariableLanguages:
|
| 53 |
+
"""`Feature` for translations with variable languages per example.
|
| 54 |
+
Here for compatibility with tfds.
|
| 55 |
+
|
| 56 |
+
Args:
|
| 57 |
+
languages (`dict`):
|
| 58 |
+
A dictionary for each example mapping string language codes to one or more string translations.
|
| 59 |
+
The languages present may vary from example to example.
|
| 60 |
+
|
| 61 |
+
Returns:
|
| 62 |
+
- `language` or `translation` (variable-length 1D `tf.Tensor` of `tf.string`):
|
| 63 |
+
Language codes sorted in ascending order or plain text translations, sorted to align with language codes.
|
| 64 |
+
|
| 65 |
+
Example:
|
| 66 |
+
|
| 67 |
+
```python
|
| 68 |
+
>>> # At construction time:
|
| 69 |
+
>>> datasets.features.TranslationVariableLanguages(languages=['en', 'fr', 'de'])
|
| 70 |
+
>>> # During data generation:
|
| 71 |
+
>>> yield {
|
| 72 |
+
... 'en': 'the cat',
|
| 73 |
+
... 'fr': ['le chat', 'la chatte,']
|
| 74 |
+
... 'de': 'die katze'
|
| 75 |
+
... }
|
| 76 |
+
>>> # Tensor returned :
|
| 77 |
+
>>> {
|
| 78 |
+
... 'language': ['en', 'de', 'fr', 'fr'],
|
| 79 |
+
... 'translation': ['the cat', 'die katze', 'la chatte', 'le chat'],
|
| 80 |
+
... }
|
| 81 |
+
```
|
| 82 |
+
"""
|
| 83 |
+
|
| 84 |
+
languages: Optional[list] = None
|
| 85 |
+
num_languages: Optional[int] = None
|
| 86 |
+
id: Optional[str] = None
|
| 87 |
+
# Automatically constructed
|
| 88 |
+
dtype: ClassVar[str] = "dict"
|
| 89 |
+
pa_type: ClassVar[Any] = None
|
| 90 |
+
_type: str = field(default="TranslationVariableLanguages", init=False, repr=False)
|
| 91 |
+
|
| 92 |
+
def __post_init__(self):
|
| 93 |
+
self.languages = sorted(set(self.languages)) if self.languages else None
|
| 94 |
+
self.num_languages = len(self.languages) if self.languages else None
|
| 95 |
+
|
| 96 |
+
def __call__(self):
|
| 97 |
+
return pa.struct({"language": pa.list_(pa.string()), "translation": pa.list_(pa.string())})
|
| 98 |
+
|
| 99 |
+
def encode_example(self, translation_dict):
|
| 100 |
+
lang_set = set(self.languages)
|
| 101 |
+
if set(translation_dict) == {"language", "translation"}:
|
| 102 |
+
return translation_dict
|
| 103 |
+
elif self.languages and set(translation_dict) - lang_set:
|
| 104 |
+
raise ValueError(
|
| 105 |
+
f"Some languages in example ({', '.join(sorted(set(translation_dict) - lang_set))}) are not in valid set ({', '.join(lang_set)})."
|
| 106 |
+
)
|
| 107 |
+
|
| 108 |
+
# Convert dictionary into tuples, splitting out cases where there are
|
| 109 |
+
# multiple translations for a single language.
|
| 110 |
+
translation_tuples = []
|
| 111 |
+
for lang, text in translation_dict.items():
|
| 112 |
+
if isinstance(text, str):
|
| 113 |
+
translation_tuples.append((lang, text))
|
| 114 |
+
else:
|
| 115 |
+
translation_tuples.extend([(lang, el) for el in text])
|
| 116 |
+
|
| 117 |
+
# Ensure translations are in ascending order by language code.
|
| 118 |
+
languages, translations = zip(*sorted(translation_tuples))
|
| 119 |
+
|
| 120 |
+
return {"language": languages, "translation": translations}
|
| 121 |
+
|
| 122 |
+
def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]:
|
| 123 |
+
"""Flatten the TranslationVariableLanguages feature into a dictionary."""
|
| 124 |
+
from .features import Sequence, Value
|
| 125 |
+
|
| 126 |
+
return {
|
| 127 |
+
"language": Sequence(Value("string")),
|
| 128 |
+
"translation": Sequence(Value("string")),
|
| 129 |
+
}
|
lib/python3.12/site-packages/datasets/features/video.py
ADDED
|
@@ -0,0 +1,280 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from dataclasses import dataclass, field
|
| 3 |
+
from typing import TYPE_CHECKING, Any, ClassVar, Optional, TypedDict, Union
|
| 4 |
+
|
| 5 |
+
import numpy as np
|
| 6 |
+
import pyarrow as pa
|
| 7 |
+
|
| 8 |
+
from .. import config
|
| 9 |
+
from ..download.download_config import DownloadConfig
|
| 10 |
+
from ..table import array_cast
|
| 11 |
+
from ..utils.file_utils import is_local_path, xopen
|
| 12 |
+
from ..utils.py_utils import string_to_dict
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
if TYPE_CHECKING:
|
| 16 |
+
from torchvision.io import VideoReader
|
| 17 |
+
|
| 18 |
+
from .features import FeatureType
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class Example(TypedDict):
|
| 22 |
+
path: Optional[str]
|
| 23 |
+
bytes: Optional[bytes]
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
@dataclass
|
| 27 |
+
class Video:
|
| 28 |
+
"""
|
| 29 |
+
**Experimental.** Video [`Feature`] to read video data from a video file.
|
| 30 |
+
|
| 31 |
+
Input: The Video feature accepts as input:
|
| 32 |
+
- A `str`: Absolute path to the video file (i.e. random access is allowed).
|
| 33 |
+
- A `dict` with the keys:
|
| 34 |
+
|
| 35 |
+
- `path`: String with relative path of the video file in a dataset repository.
|
| 36 |
+
- `bytes`: Bytes of the video file.
|
| 37 |
+
|
| 38 |
+
This is useful for archived files with sequential access.
|
| 39 |
+
|
| 40 |
+
- A `torchvision.io.VideoReader`: torchvision video reader object.
|
| 41 |
+
|
| 42 |
+
Args:
|
| 43 |
+
mode (`str`, *optional*):
|
| 44 |
+
The mode to convert the video to. If `None`, the native mode of the video is used.
|
| 45 |
+
decode (`bool`, defaults to `True`):
|
| 46 |
+
Whether to decode the video data. If `False`,
|
| 47 |
+
returns the underlying dictionary in the format `{"path": video_path, "bytes": video_bytes}`.
|
| 48 |
+
|
| 49 |
+
Examples:
|
| 50 |
+
|
| 51 |
+
```py
|
| 52 |
+
>>> from datasets import Dataset, Video
|
| 53 |
+
>>> ds = Dataset.from_dict({"video":["path/to/Screen Recording.mov"]}).cast_column("video", Video())
|
| 54 |
+
>>> ds.features["video"]
|
| 55 |
+
Video(decode=True, id=None)
|
| 56 |
+
>>> ds[0]["video"]
|
| 57 |
+
<torchvision.io.video_reader.VideoReader object at 0x325b1aae0>
|
| 58 |
+
>>> ds = ds.cast_column('video', Video(decode=False))
|
| 59 |
+
{'bytes': None,
|
| 60 |
+
'path': 'path/to/Screen Recording.mov'}
|
| 61 |
+
```
|
| 62 |
+
"""
|
| 63 |
+
|
| 64 |
+
decode: bool = True
|
| 65 |
+
id: Optional[str] = None
|
| 66 |
+
# Automatically constructed
|
| 67 |
+
dtype: ClassVar[str] = "torchvision.io.VideoReader"
|
| 68 |
+
pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()})
|
| 69 |
+
_type: str = field(default="Video", init=False, repr=False)
|
| 70 |
+
|
| 71 |
+
def __call__(self):
|
| 72 |
+
return self.pa_type
|
| 73 |
+
|
| 74 |
+
def encode_example(self, value: Union[str, bytes, bytearray, Example, np.ndarray, "VideoReader"]) -> Example:
|
| 75 |
+
"""Encode example into a format for Arrow.
|
| 76 |
+
|
| 77 |
+
Args:
|
| 78 |
+
value (`str`, `np.ndarray`, `VideoReader` or `dict`):
|
| 79 |
+
Data passed as input to Video feature.
|
| 80 |
+
|
| 81 |
+
Returns:
|
| 82 |
+
`dict` with "path" and "bytes" fields
|
| 83 |
+
"""
|
| 84 |
+
if config.TORCHVISION_AVAILABLE:
|
| 85 |
+
from torchvision.io import VideoReader
|
| 86 |
+
|
| 87 |
+
else:
|
| 88 |
+
VideoReader = None
|
| 89 |
+
|
| 90 |
+
if isinstance(value, list):
|
| 91 |
+
value = np.array(value)
|
| 92 |
+
|
| 93 |
+
if isinstance(value, str):
|
| 94 |
+
return {"path": value, "bytes": None}
|
| 95 |
+
elif isinstance(value, (bytes, bytearray)):
|
| 96 |
+
return {"path": None, "bytes": value}
|
| 97 |
+
elif isinstance(value, np.ndarray):
|
| 98 |
+
# convert the video array to bytes
|
| 99 |
+
return encode_np_array(value)
|
| 100 |
+
elif VideoReader is not None and isinstance(value, VideoReader):
|
| 101 |
+
# convert the torchvision video reader to bytes
|
| 102 |
+
return encode_torchvision_video(value)
|
| 103 |
+
elif isinstance(value, dict):
|
| 104 |
+
path, bytes_ = value.get("path"), value.get("bytes")
|
| 105 |
+
if path is not None and os.path.isfile(path):
|
| 106 |
+
# we set "bytes": None to not duplicate the data if they're already available locally
|
| 107 |
+
return {"bytes": None, "path": path}
|
| 108 |
+
elif bytes_ is not None or path is not None:
|
| 109 |
+
# store the video bytes, and path is used to infer the video format using the file extension
|
| 110 |
+
return {"bytes": bytes_, "path": path}
|
| 111 |
+
else:
|
| 112 |
+
raise ValueError(
|
| 113 |
+
f"A video sample should have one of 'path' or 'bytes' but they are missing or None in {value}."
|
| 114 |
+
)
|
| 115 |
+
else:
|
| 116 |
+
raise TypeError(f"Unsupported encode_example type: {type(value)}")
|
| 117 |
+
|
| 118 |
+
def decode_example(
|
| 119 |
+
self,
|
| 120 |
+
value: Union[str, Example],
|
| 121 |
+
token_per_repo_id: Optional[dict[str, Union[bool, str]]] = None,
|
| 122 |
+
) -> "VideoReader":
|
| 123 |
+
"""Decode example video file into video data.
|
| 124 |
+
|
| 125 |
+
Args:
|
| 126 |
+
value (`str` or `dict`):
|
| 127 |
+
A string with the absolute video file path, a dictionary with
|
| 128 |
+
keys:
|
| 129 |
+
|
| 130 |
+
- `path`: String with absolute or relative video file path.
|
| 131 |
+
- `bytes`: The bytes of the video file.
|
| 132 |
+
token_per_repo_id (`dict`, *optional*):
|
| 133 |
+
To access and decode
|
| 134 |
+
video files from private repositories on the Hub, you can pass
|
| 135 |
+
a dictionary repo_id (`str`) -> token (`bool` or `str`).
|
| 136 |
+
|
| 137 |
+
Returns:
|
| 138 |
+
`torchvision.io.VideoReader`
|
| 139 |
+
"""
|
| 140 |
+
if not self.decode:
|
| 141 |
+
raise RuntimeError("Decoding is disabled for this feature. Please use Video(decode=True) instead.")
|
| 142 |
+
|
| 143 |
+
if config.TORCHVISION_AVAILABLE:
|
| 144 |
+
from torchvision.io import VideoReader
|
| 145 |
+
|
| 146 |
+
else:
|
| 147 |
+
raise ImportError("To support decoding videos, please install 'torchvision'.")
|
| 148 |
+
|
| 149 |
+
if token_per_repo_id is None:
|
| 150 |
+
token_per_repo_id = {}
|
| 151 |
+
|
| 152 |
+
if isinstance(value, str):
|
| 153 |
+
path, bytes_ = value, None
|
| 154 |
+
else:
|
| 155 |
+
path, bytes_ = value["path"], value["bytes"]
|
| 156 |
+
|
| 157 |
+
if bytes_ is None:
|
| 158 |
+
if path is None:
|
| 159 |
+
raise ValueError(f"A video should have one of 'path' or 'bytes' but both are None in {value}.")
|
| 160 |
+
elif is_local_path(path):
|
| 161 |
+
video = VideoReader(path)
|
| 162 |
+
else:
|
| 163 |
+
video = hf_video_reader(path, token_per_repo_id=token_per_repo_id)
|
| 164 |
+
else:
|
| 165 |
+
video = VideoReader(bytes_)
|
| 166 |
+
video._hf_encoded = {"path": path, "bytes": bytes_}
|
| 167 |
+
return video
|
| 168 |
+
|
| 169 |
+
def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]:
|
| 170 |
+
"""If in the decodable state, return the feature itself, otherwise flatten the feature into a dictionary."""
|
| 171 |
+
from .features import Value
|
| 172 |
+
|
| 173 |
+
return (
|
| 174 |
+
self
|
| 175 |
+
if self.decode
|
| 176 |
+
else {
|
| 177 |
+
"bytes": Value("binary"),
|
| 178 |
+
"path": Value("string"),
|
| 179 |
+
}
|
| 180 |
+
)
|
| 181 |
+
|
| 182 |
+
def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray, pa.ListArray]) -> pa.StructArray:
|
| 183 |
+
"""Cast an Arrow array to the Video arrow storage type.
|
| 184 |
+
The Arrow types that can be converted to the Video pyarrow storage type are:
|
| 185 |
+
|
| 186 |
+
- `pa.string()` - it must contain the "path" data
|
| 187 |
+
- `pa.binary()` - it must contain the video bytes
|
| 188 |
+
- `pa.struct({"bytes": pa.binary()})`
|
| 189 |
+
- `pa.struct({"path": pa.string()})`
|
| 190 |
+
- `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter
|
| 191 |
+
- `pa.list(*)` - it must contain the video array data
|
| 192 |
+
|
| 193 |
+
Args:
|
| 194 |
+
storage (`Union[pa.StringArray, pa.StructArray, pa.ListArray]`):
|
| 195 |
+
PyArrow array to cast.
|
| 196 |
+
|
| 197 |
+
Returns:
|
| 198 |
+
`pa.StructArray`: Array in the Video arrow storage type, that is
|
| 199 |
+
`pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
|
| 200 |
+
"""
|
| 201 |
+
if pa.types.is_string(storage.type):
|
| 202 |
+
bytes_array = pa.array([None] * len(storage), type=pa.binary())
|
| 203 |
+
storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null())
|
| 204 |
+
elif pa.types.is_binary(storage.type):
|
| 205 |
+
path_array = pa.array([None] * len(storage), type=pa.string())
|
| 206 |
+
storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null())
|
| 207 |
+
elif pa.types.is_struct(storage.type):
|
| 208 |
+
if storage.type.get_field_index("bytes") >= 0:
|
| 209 |
+
bytes_array = storage.field("bytes")
|
| 210 |
+
else:
|
| 211 |
+
bytes_array = pa.array([None] * len(storage), type=pa.binary())
|
| 212 |
+
if storage.type.get_field_index("path") >= 0:
|
| 213 |
+
path_array = storage.field("path")
|
| 214 |
+
else:
|
| 215 |
+
path_array = pa.array([None] * len(storage), type=pa.string())
|
| 216 |
+
storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null())
|
| 217 |
+
elif pa.types.is_list(storage.type):
|
| 218 |
+
bytes_array = pa.array(
|
| 219 |
+
[encode_np_array(np.array(arr))["bytes"] if arr is not None else None for arr in storage.to_pylist()],
|
| 220 |
+
type=pa.binary(),
|
| 221 |
+
)
|
| 222 |
+
path_array = pa.array([None] * len(storage), type=pa.string())
|
| 223 |
+
storage = pa.StructArray.from_arrays(
|
| 224 |
+
[bytes_array, path_array], ["bytes", "path"], mask=bytes_array.is_null()
|
| 225 |
+
)
|
| 226 |
+
return array_cast(storage, self.pa_type)
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
def video_to_bytes(video: "VideoReader") -> bytes:
|
| 230 |
+
"""Convert a torchvision Video object to bytes using native compression if possible"""
|
| 231 |
+
raise NotImplementedError()
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def encode_torchvision_video(video: "VideoReader") -> Example:
|
| 235 |
+
if hasattr(video, "_hf_encoded"):
|
| 236 |
+
return video._hf_encoded
|
| 237 |
+
else:
|
| 238 |
+
raise NotImplementedError(
|
| 239 |
+
"Encoding a VideoReader that doesn't come from datasets.Video.decode() is not implemented"
|
| 240 |
+
)
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
def encode_np_array(array: np.ndarray) -> Example:
|
| 244 |
+
raise NotImplementedError()
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
# Patching torchvision a little bit to:
|
| 248 |
+
# 1. store the encoded video data {"path": ..., "bytes": ...} in `video._hf_encoded``
|
| 249 |
+
# 2. add support for hf:// files
|
| 250 |
+
# This doesn't affect the normal usage of torchvision.
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
def hf_video_reader(
|
| 254 |
+
path: str, token_per_repo_id: Optional[dict[str, Union[bool, str]]] = None, stream: str = "video"
|
| 255 |
+
) -> "VideoReader":
|
| 256 |
+
import av
|
| 257 |
+
from torchvision import get_video_backend
|
| 258 |
+
from torchvision.io import VideoReader
|
| 259 |
+
|
| 260 |
+
# Load the file from HF
|
| 261 |
+
if token_per_repo_id is None:
|
| 262 |
+
token_per_repo_id = {}
|
| 263 |
+
source_url = path.split("::")[-1]
|
| 264 |
+
pattern = config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL
|
| 265 |
+
source_url_fields = string_to_dict(source_url, pattern)
|
| 266 |
+
token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None
|
| 267 |
+
download_config = DownloadConfig(token=token)
|
| 268 |
+
f = xopen(path, "rb", download_config=download_config)
|
| 269 |
+
|
| 270 |
+
# Instantiate the VideoReader
|
| 271 |
+
vr = object.__new__(VideoReader)
|
| 272 |
+
vr.backend = get_video_backend()
|
| 273 |
+
if vr.backend != "pyav":
|
| 274 |
+
raise RuntimeError(f"Unsupported video backend for VideoReader from HF files: {vr.backend}")
|
| 275 |
+
vr.container = av.open(f, metadata_errors="ignore")
|
| 276 |
+
stream_type = stream.split(":")[0]
|
| 277 |
+
stream_id = 0 if len(stream.split(":")) == 1 else int(stream.split(":")[1])
|
| 278 |
+
vr.pyav_stream = {stream_type: stream_id}
|
| 279 |
+
vr._c = vr.container.decode(**vr.pyav_stream)
|
| 280 |
+
return vr
|
lib/python3.12/site-packages/datasets/filesystems/__init__.py
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import importlib
|
| 2 |
+
import shutil
|
| 3 |
+
import warnings
|
| 4 |
+
from typing import List
|
| 5 |
+
|
| 6 |
+
import fsspec
|
| 7 |
+
import fsspec.asyn
|
| 8 |
+
from fsspec.implementations.local import LocalFileSystem
|
| 9 |
+
|
| 10 |
+
from . import compression
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
COMPRESSION_FILESYSTEMS: list[compression.BaseCompressedFileFileSystem] = [
|
| 14 |
+
compression.Bz2FileSystem,
|
| 15 |
+
compression.GzipFileSystem,
|
| 16 |
+
compression.Lz4FileSystem,
|
| 17 |
+
compression.XzFileSystem,
|
| 18 |
+
compression.ZstdFileSystem,
|
| 19 |
+
]
|
| 20 |
+
|
| 21 |
+
# Register custom filesystems
|
| 22 |
+
for fs_class in COMPRESSION_FILESYSTEMS:
|
| 23 |
+
if fs_class.protocol in fsspec.registry and fsspec.registry[fs_class.protocol] is not fs_class:
|
| 24 |
+
warnings.warn(f"A filesystem protocol was already set for {fs_class.protocol} and will be overwritten.")
|
| 25 |
+
fsspec.register_implementation(fs_class.protocol, fs_class, clobber=True)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def is_remote_filesystem(fs: fsspec.AbstractFileSystem) -> bool:
|
| 29 |
+
"""
|
| 30 |
+
Checks if `fs` is a remote filesystem.
|
| 31 |
+
|
| 32 |
+
Args:
|
| 33 |
+
fs (`fsspec.spec.AbstractFileSystem`):
|
| 34 |
+
An abstract super-class for pythonic file-systems, e.g. `fsspec.filesystem(\'file\')` or `s3fs.S3FileSystem`.
|
| 35 |
+
"""
|
| 36 |
+
return not isinstance(fs, LocalFileSystem)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def rename(fs: fsspec.AbstractFileSystem, src: str, dst: str):
|
| 40 |
+
"""
|
| 41 |
+
Renames the file `src` in `fs` to `dst`.
|
| 42 |
+
"""
|
| 43 |
+
if not is_remote_filesystem(fs):
|
| 44 |
+
# LocalFileSystem.mv does copy + rm, it is more efficient to simply move a local directory
|
| 45 |
+
shutil.move(fs._strip_protocol(src), fs._strip_protocol(dst))
|
| 46 |
+
else:
|
| 47 |
+
fs.mv(src, dst, recursive=True)
|
lib/python3.12/site-packages/datasets/filesystems/__pycache__/__init__.cpython-312.pyc
ADDED
|
Binary file (2.35 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/filesystems/__pycache__/compression.cpython-312.pyc
ADDED
|
Binary file (6.04 kB). View file
|
|
|
lib/python3.12/site-packages/datasets/filesystems/compression.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from functools import partial
|
| 3 |
+
from typing import Optional
|
| 4 |
+
|
| 5 |
+
import fsspec
|
| 6 |
+
from fsspec.archive import AbstractArchiveFileSystem
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class BaseCompressedFileFileSystem(AbstractArchiveFileSystem):
|
| 10 |
+
"""Read contents of compressed file as a filesystem with one file inside."""
|
| 11 |
+
|
| 12 |
+
root_marker = ""
|
| 13 |
+
protocol: str = (
|
| 14 |
+
None # protocol passed in prefix to the url. ex: "gzip", for gzip://file.txt::http://foo.bar/file.txt.gz
|
| 15 |
+
)
|
| 16 |
+
compression: str = None # compression type in fsspec. ex: "gzip"
|
| 17 |
+
extension: str = None # extension of the filename to strip. ex: "".gz" to get file.txt from file.txt.gz
|
| 18 |
+
|
| 19 |
+
def __init__(
|
| 20 |
+
self, fo: str = "", target_protocol: Optional[str] = None, target_options: Optional[dict] = None, **kwargs
|
| 21 |
+
):
|
| 22 |
+
"""
|
| 23 |
+
The compressed file system can be instantiated from any compressed file.
|
| 24 |
+
It reads the contents of compressed file as a filesystem with one file inside, as if it was an archive.
|
| 25 |
+
|
| 26 |
+
The single file inside the filesystem is named after the compresssed file,
|
| 27 |
+
without the compression extension at the end of the filename.
|
| 28 |
+
|
| 29 |
+
Args:
|
| 30 |
+
fo (:obj:``str``): Path to compressed file. Will fetch file using ``fsspec.open()``
|
| 31 |
+
mode (:obj:``str``): Currently, only 'rb' accepted
|
| 32 |
+
target_protocol(:obj:``str``, optional): To override the FS protocol inferred from a URL.
|
| 33 |
+
target_options (:obj:``dict``, optional): Kwargs passed when instantiating the target FS.
|
| 34 |
+
"""
|
| 35 |
+
super().__init__(self, **kwargs)
|
| 36 |
+
self.fo = fo.__fspath__() if hasattr(fo, "__fspath__") else fo
|
| 37 |
+
# always open as "rb" since fsspec can then use the TextIOWrapper to make it work for "r" mode
|
| 38 |
+
self._open_with_fsspec = partial(
|
| 39 |
+
fsspec.open,
|
| 40 |
+
self.fo,
|
| 41 |
+
mode="rb",
|
| 42 |
+
protocol=target_protocol,
|
| 43 |
+
compression=self.compression,
|
| 44 |
+
client_kwargs={
|
| 45 |
+
"requote_redirect_url": False, # see https://github.com/huggingface/datasets/pull/5459
|
| 46 |
+
"trust_env": True, # Enable reading proxy env variables.
|
| 47 |
+
**(target_options or {}).pop("client_kwargs", {}), # To avoid issues if it was already passed.
|
| 48 |
+
},
|
| 49 |
+
**(target_options or {}),
|
| 50 |
+
)
|
| 51 |
+
self.compressed_name = os.path.basename(self.fo.split("::")[0])
|
| 52 |
+
self.uncompressed_name = (
|
| 53 |
+
self.compressed_name[: self.compressed_name.rindex(".")]
|
| 54 |
+
if "." in self.compressed_name
|
| 55 |
+
else self.compressed_name
|
| 56 |
+
)
|
| 57 |
+
self.dir_cache = None
|
| 58 |
+
|
| 59 |
+
@classmethod
|
| 60 |
+
def _strip_protocol(cls, path):
|
| 61 |
+
# compressed file paths are always relative to the archive root
|
| 62 |
+
return super()._strip_protocol(path).lstrip("/")
|
| 63 |
+
|
| 64 |
+
def _get_dirs(self):
|
| 65 |
+
if self.dir_cache is None:
|
| 66 |
+
f = {**self._open_with_fsspec().fs.info(self.fo), "name": self.uncompressed_name}
|
| 67 |
+
self.dir_cache = {f["name"]: f}
|
| 68 |
+
|
| 69 |
+
def cat(self, path: str):
|
| 70 |
+
with self._open_with_fsspec().open() as f:
|
| 71 |
+
return f.read()
|
| 72 |
+
|
| 73 |
+
def _open(
|
| 74 |
+
self,
|
| 75 |
+
path: str,
|
| 76 |
+
mode: str = "rb",
|
| 77 |
+
block_size=None,
|
| 78 |
+
autocommit=True,
|
| 79 |
+
cache_options=None,
|
| 80 |
+
**kwargs,
|
| 81 |
+
):
|
| 82 |
+
path = self._strip_protocol(path)
|
| 83 |
+
if mode != "rb":
|
| 84 |
+
raise ValueError(f"Tried to read with mode {mode} on file {self.fo} opened with mode 'rb'")
|
| 85 |
+
return self._open_with_fsspec().open()
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
class Bz2FileSystem(BaseCompressedFileFileSystem):
|
| 89 |
+
"""Read contents of BZ2 file as a filesystem with one file inside."""
|
| 90 |
+
|
| 91 |
+
protocol = "bz2"
|
| 92 |
+
compression = "bz2"
|
| 93 |
+
extension = ".bz2"
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
class GzipFileSystem(BaseCompressedFileFileSystem):
|
| 97 |
+
"""Read contents of GZIP file as a filesystem with one file inside."""
|
| 98 |
+
|
| 99 |
+
protocol = "gzip"
|
| 100 |
+
compression = "gzip"
|
| 101 |
+
extension = ".gz"
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
class Lz4FileSystem(BaseCompressedFileFileSystem):
|
| 105 |
+
"""Read contents of LZ4 file as a filesystem with one file inside."""
|
| 106 |
+
|
| 107 |
+
protocol = "lz4"
|
| 108 |
+
compression = "lz4"
|
| 109 |
+
extension = ".lz4"
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
class XzFileSystem(BaseCompressedFileFileSystem):
|
| 113 |
+
"""Read contents of .xz (LZMA) file as a filesystem with one file inside."""
|
| 114 |
+
|
| 115 |
+
protocol = "xz"
|
| 116 |
+
compression = "xz"
|
| 117 |
+
extension = ".xz"
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
class ZstdFileSystem(BaseCompressedFileFileSystem):
|
| 121 |
+
"""
|
| 122 |
+
Read contents of .zstd file as a filesystem with one file inside.
|
| 123 |
+
"""
|
| 124 |
+
|
| 125 |
+
protocol = "zstd"
|
| 126 |
+
compression = "zstd"
|
| 127 |
+
extension = ".zst"
|