diff --git a/.gitattributes b/.gitattributes
index 263c8f7009bdd24ae6146f4fea4f236de700a300..ee062d3ed7a83280012895e05318febbebf881a9 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -50,3 +50,7 @@ venv/Lib/site-packages/cv2/opencv_videoio_ffmpeg4130_64.dll filter=lfs diff=lfs
venv/Lib/site-packages/datasets/__pycache__/arrow_dataset.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
venv/Lib/site-packages/datasets/__pycache__/builder.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
venv/Lib/site-packages/datasets/__pycache__/dataset_dict.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
+venv/Lib/site-packages/datasets/__pycache__/iterable_dataset.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
+venv/Lib/site-packages/datasets/__pycache__/table.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
+venv/Lib/site-packages/datasets/features/__pycache__/features.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
+venv/Lib/site-packages/dill/__pycache__/_dill.cpython-311.pyc filter=lfs diff=lfs merge=lfs -text
diff --git a/venv/Lib/site-packages/datasets-5.0.0.dist-info/AUTHORS b/venv/Lib/site-packages/datasets-5.0.0.dist-info/AUTHORS
new file mode 100644
index 0000000000000000000000000000000000000000..7092efd949bb407beb485f25d6c1847804a79170
--- /dev/null
+++ b/venv/Lib/site-packages/datasets-5.0.0.dist-info/AUTHORS
@@ -0,0 +1,8 @@
+# This is the list of HuggingFace Datasets authors for copyright purposes.
+#
+# This does not necessarily list everyone who has contributed code, since in
+# some cases, their employer may be the copyright holder. To see the full list
+# of contributors, see the revision history in source control.
+
+Google Inc.
+HuggingFace Inc.
diff --git a/venv/Lib/site-packages/datasets-5.0.0.dist-info/INSTALLER b/venv/Lib/site-packages/datasets-5.0.0.dist-info/INSTALLER
new file mode 100644
index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68
--- /dev/null
+++ b/venv/Lib/site-packages/datasets-5.0.0.dist-info/INSTALLER
@@ -0,0 +1 @@
+pip
diff --git a/venv/Lib/site-packages/datasets-5.0.0.dist-info/LICENSE b/venv/Lib/site-packages/datasets-5.0.0.dist-info/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..d645695673349e3947e8e5ae42332d0ac3164cd7
--- /dev/null
+++ b/venv/Lib/site-packages/datasets-5.0.0.dist-info/LICENSE
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/venv/Lib/site-packages/datasets-5.0.0.dist-info/METADATA b/venv/Lib/site-packages/datasets-5.0.0.dist-info/METADATA
new file mode 100644
index 0000000000000000000000000000000000000000..88ae5f7595eca7ef895bf8345999d38872425dea
--- /dev/null
+++ b/venv/Lib/site-packages/datasets-5.0.0.dist-info/METADATA
@@ -0,0 +1,500 @@
+Metadata-Version: 2.2
+Name: datasets
+Version: 5.0.0
+Summary: HuggingFace community-driven open-source library of datasets
+Home-page: https://github.com/huggingface/datasets
+Download-URL: https://github.com/huggingface/datasets/tags
+Author: HuggingFace Inc.
+Author-email: thomas@huggingface.co
+License: Apache 2.0
+Keywords: datasets machine learning datasets
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Education
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10.0
+Description-Content-Type: text/markdown
+License-File: LICENSE
+License-File: AUTHORS
+Requires-Dist: filelock
+Requires-Dist: numpy>=1.17
+Requires-Dist: pyarrow>=21.0.0
+Requires-Dist: dill<0.4.2,>=0.3.0
+Requires-Dist: pandas
+Requires-Dist: requests>=2.32.2
+Requires-Dist: httpx<1.0.0
+Requires-Dist: tqdm>=4.66.3
+Requires-Dist: xxhash
+Requires-Dist: multiprocess<0.70.20
+Requires-Dist: fsspec[http]<=2026.4.0,>=2023.1.0
+Requires-Dist: huggingface-hub<2.0,>=0.25.0
+Requires-Dist: packaging
+Requires-Dist: pyyaml>=5.1
+Provides-Extra: audio
+Requires-Dist: torchcodec>=0.6.0; extra == "audio"
+Requires-Dist: torch>=2.8.0; extra == "audio"
+Provides-Extra: vision
+Requires-Dist: Pillow>=9.4.0; extra == "vision"
+Provides-Extra: mesh
+Requires-Dist: trimesh>=4.10.0; extra == "mesh"
+Provides-Extra: tensorflow
+Requires-Dist: tensorflow>=2.6.0; extra == "tensorflow"
+Provides-Extra: tensorflow-gpu
+Requires-Dist: tensorflow>=2.6.0; extra == "tensorflow-gpu"
+Provides-Extra: torch
+Requires-Dist: torch; extra == "torch"
+Provides-Extra: jax
+Requires-Dist: jax>=0.3.14; extra == "jax"
+Requires-Dist: jaxlib>=0.3.14; extra == "jax"
+Provides-Extra: streaming
+Provides-Extra: dev
+Requires-Dist: numba>=0.56.4; python_version < "3.14" and extra == "dev"
+Requires-Dist: absl-py; extra == "dev"
+Requires-Dist: decorator; extra == "dev"
+Requires-Dist: joblib<1.3.0; extra == "dev"
+Requires-Dist: joblibspark; python_version < "3.14" and extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-datadir; extra == "dev"
+Requires-Dist: pytest-xdist; extra == "dev"
+Requires-Dist: aiohttp; extra == "dev"
+Requires-Dist: elasticsearch<8.0.0,>=7.17.12; extra == "dev"
+Requires-Dist: faiss-cpu>=1.8.0.post1; extra == "dev"
+Requires-Dist: h5py; extra == "dev"
+Requires-Dist: pylance; extra == "dev"
+Requires-Dist: pyiceberg[pyarrow,sql-sqlite]; extra == "dev"
+Requires-Dist: jax>=0.3.14; sys_platform != "win32" and extra == "dev"
+Requires-Dist: jaxlib>=0.3.14; sys_platform != "win32" and extra == "dev"
+Requires-Dist: lz4; python_version < "3.14" and extra == "dev"
+Requires-Dist: moto[server]; extra == "dev"
+Requires-Dist: pyspark>=3.4; extra == "dev"
+Requires-Dist: py7zr; extra == "dev"
+Requires-Dist: rarfile>=4.0; extra == "dev"
+Requires-Dist: sqlalchemy; extra == "dev"
+Requires-Dist: protobuf<4.0.0; extra == "dev"
+Requires-Dist: tensorflow>=2.6.0; (python_version < "3.10" and sys_platform != "win32") and extra == "dev"
+Requires-Dist: tensorflow>=2.16.0; (python_version >= "3.10" and sys_platform != "win32" and python_version < "3.14") and extra == "dev"
+Requires-Dist: tiktoken; extra == "dev"
+Requires-Dist: torch>=2.8.0; extra == "dev"
+Requires-Dist: torchdata; extra == "dev"
+Requires-Dist: transformers>=4.42.0; extra == "dev"
+Requires-Dist: zstandard; extra == "dev"
+Requires-Dist: polars[timezone]>=0.20.0; extra == "dev"
+Requires-Dist: Pillow>=9.4.0; extra == "dev"
+Requires-Dist: torchcodec>=0.7.0; python_version < "3.14" and extra == "dev"
+Requires-Dist: nibabel>=5.3.1; extra == "dev"
+Requires-Dist: trimesh>=4.10.0; extra == "dev"
+Requires-Dist: teich==0.1.1a76; extra == "dev"
+Requires-Dist: ruff>=0.3.0; extra == "dev"
+Requires-Dist: transformers; extra == "dev"
+Requires-Dist: torch; extra == "dev"
+Requires-Dist: tensorflow>=2.6.0; extra == "dev"
+Provides-Extra: tests
+Requires-Dist: numba>=0.56.4; python_version < "3.14" and extra == "tests"
+Requires-Dist: absl-py; extra == "tests"
+Requires-Dist: decorator; extra == "tests"
+Requires-Dist: joblib<1.3.0; extra == "tests"
+Requires-Dist: joblibspark; python_version < "3.14" and extra == "tests"
+Requires-Dist: pytest; extra == "tests"
+Requires-Dist: pytest-datadir; extra == "tests"
+Requires-Dist: pytest-xdist; extra == "tests"
+Requires-Dist: aiohttp; extra == "tests"
+Requires-Dist: elasticsearch<8.0.0,>=7.17.12; extra == "tests"
+Requires-Dist: faiss-cpu>=1.8.0.post1; extra == "tests"
+Requires-Dist: h5py; extra == "tests"
+Requires-Dist: pylance; extra == "tests"
+Requires-Dist: pyiceberg[pyarrow,sql-sqlite]; extra == "tests"
+Requires-Dist: jax>=0.3.14; sys_platform != "win32" and extra == "tests"
+Requires-Dist: jaxlib>=0.3.14; sys_platform != "win32" and extra == "tests"
+Requires-Dist: lz4; python_version < "3.14" and extra == "tests"
+Requires-Dist: moto[server]; extra == "tests"
+Requires-Dist: pyspark>=3.4; extra == "tests"
+Requires-Dist: py7zr; extra == "tests"
+Requires-Dist: rarfile>=4.0; extra == "tests"
+Requires-Dist: sqlalchemy; extra == "tests"
+Requires-Dist: protobuf<4.0.0; extra == "tests"
+Requires-Dist: tensorflow>=2.6.0; (python_version < "3.10" and sys_platform != "win32") and extra == "tests"
+Requires-Dist: tensorflow>=2.16.0; (python_version >= "3.10" and sys_platform != "win32" and python_version < "3.14") and extra == "tests"
+Requires-Dist: tiktoken; extra == "tests"
+Requires-Dist: torch>=2.8.0; extra == "tests"
+Requires-Dist: torchdata; extra == "tests"
+Requires-Dist: transformers>=4.42.0; extra == "tests"
+Requires-Dist: zstandard; extra == "tests"
+Requires-Dist: polars[timezone]>=0.20.0; extra == "tests"
+Requires-Dist: Pillow>=9.4.0; extra == "tests"
+Requires-Dist: torchcodec>=0.7.0; python_version < "3.14" and extra == "tests"
+Requires-Dist: nibabel>=5.3.1; extra == "tests"
+Requires-Dist: trimesh>=4.10.0; extra == "tests"
+Requires-Dist: teich==0.1.1a76; extra == "tests"
+Provides-Extra: tests-numpy2
+Requires-Dist: numba>=0.56.4; python_version < "3.14" and extra == "tests-numpy2"
+Requires-Dist: absl-py; extra == "tests-numpy2"
+Requires-Dist: decorator; extra == "tests-numpy2"
+Requires-Dist: joblib<1.3.0; extra == "tests-numpy2"
+Requires-Dist: joblibspark; python_version < "3.14" and extra == "tests-numpy2"
+Requires-Dist: pytest; extra == "tests-numpy2"
+Requires-Dist: pytest-datadir; extra == "tests-numpy2"
+Requires-Dist: pytest-xdist; extra == "tests-numpy2"
+Requires-Dist: aiohttp; extra == "tests-numpy2"
+Requires-Dist: elasticsearch<8.0.0,>=7.17.12; extra == "tests-numpy2"
+Requires-Dist: h5py; extra == "tests-numpy2"
+Requires-Dist: pylance; extra == "tests-numpy2"
+Requires-Dist: pyiceberg[pyarrow,sql-sqlite]; extra == "tests-numpy2"
+Requires-Dist: jax>=0.3.14; sys_platform != "win32" and extra == "tests-numpy2"
+Requires-Dist: jaxlib>=0.3.14; sys_platform != "win32" and extra == "tests-numpy2"
+Requires-Dist: lz4; python_version < "3.14" and extra == "tests-numpy2"
+Requires-Dist: moto[server]; extra == "tests-numpy2"
+Requires-Dist: pyspark>=3.4; extra == "tests-numpy2"
+Requires-Dist: py7zr; extra == "tests-numpy2"
+Requires-Dist: rarfile>=4.0; extra == "tests-numpy2"
+Requires-Dist: sqlalchemy; extra == "tests-numpy2"
+Requires-Dist: protobuf<4.0.0; extra == "tests-numpy2"
+Requires-Dist: tiktoken; extra == "tests-numpy2"
+Requires-Dist: torch>=2.8.0; extra == "tests-numpy2"
+Requires-Dist: torchdata; extra == "tests-numpy2"
+Requires-Dist: transformers>=4.42.0; extra == "tests-numpy2"
+Requires-Dist: zstandard; extra == "tests-numpy2"
+Requires-Dist: polars[timezone]>=0.20.0; extra == "tests-numpy2"
+Requires-Dist: Pillow>=9.4.0; extra == "tests-numpy2"
+Requires-Dist: torchcodec>=0.7.0; python_version < "3.14" and extra == "tests-numpy2"
+Requires-Dist: nibabel>=5.3.1; extra == "tests-numpy2"
+Requires-Dist: trimesh>=4.10.0; extra == "tests-numpy2"
+Requires-Dist: teich==0.1.1a76; extra == "tests-numpy2"
+Provides-Extra: quality
+Requires-Dist: ruff>=0.3.0; extra == "quality"
+Provides-Extra: benchmarks
+Requires-Dist: tensorflow==2.12.0; extra == "benchmarks"
+Requires-Dist: torch==2.0.1; extra == "benchmarks"
+Requires-Dist: transformers==4.30.1; extra == "benchmarks"
+Provides-Extra: docs
+Requires-Dist: transformers; extra == "docs"
+Requires-Dist: torch; extra == "docs"
+Requires-Dist: tensorflow>=2.6.0; extra == "docs"
+Provides-Extra: pdfs
+Requires-Dist: pdfplumber>=0.11.4; extra == "pdfs"
+Provides-Extra: nibabel
+Requires-Dist: nibabel>=5.3.2; extra == "nibabel"
+Requires-Dist: ipyniivue==2.4.2; extra == "nibabel"
+Provides-Extra: iceberg
+Requires-Dist: pyiceberg>=0.7.0; extra == "iceberg"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: download-url
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+🤗 Datasets is a lightweight library providing **two** main features:
+
+- **one-line dataloaders for many public datasets**: one-liners to download and pre-process any of the  major public datasets (image datasets, audio datasets, text datasets in 467 languages and dialects, 3D medical images, video datasets, agent traces, etc.) provided on the [HuggingFace Datasets Hub](https://huggingface.co/datasets). With a simple command like `squad_dataset = load_dataset("rajpurkar/squad")`, get any of these datasets ready to use in a dataloader for training/evaluating a ML model (Numpy/Pandas/PyTorch/TensorFlow/JAX/Polars),
+- **efficient data pre-processing**: simple, fast and reproducible data pre-processing for the public datasets as well as your own local datasets in CSV, JSON, JSONL, Parquet, HDF5, XML, text, PNG, JPEG, WAV, MP3, PDF, NIfTI, and more. With simple commands like `processed_dataset = dataset.map(process_example)`, efficiently prepare the dataset for inspection and ML model evaluation and training.
+
+[🎓 **Documentation**](https://huggingface.co/docs/datasets/) [🔎 **Find a dataset in the Hub**](https://huggingface.co/datasets) [🌟 **Share a dataset on the Hub**](https://huggingface.co/docs/datasets/share)
+
+
+
+
+
+# 🚀 Key Features
+
+🤗 Datasets is designed to let the community easily add and share new datasets, and provides powerful capabilities for data manipulation:
+
+| Feature | Description |
+|---------|-------------|
+| 📦 **One-line dataset loading** | Load AI-ready datasets from the [Hugging Face Hub](https://huggingface.co/datasets) or local files with `load_dataset()` |
+| 🔍 **Multiple formats** | Native support for CSV, JSON, JSONL, Parquet, Arrow, XML, Text, Webdataset, and more |
+| 🖼️ **Multi-modal data** | Built-in support for text, audio, image, video, PDF, and NIfTI (3D medical) data |
+| 🚀 **Streaming mode** | Stream datasets without downloading — iterate over data on-the-fly with `streaming=True` (now up to **100x faster** with Xet backend) |
+| 💾 **HF Storage Buckets** | Read and write directly from/to [Hugging Face Storage Buckets](https://huggingface.co/docs/hub/storage-buckets) for mutable, large-scale raw data |
+| 🧠 **AI Agent Traces** | Load and process AI agent traces (prompts, tool calls, responses) from the Hub |
+| ⚡ **Apache Arrow backend** | Zero-copy memory-mapped storage — datasets naturally free you from RAM limitations |
+| 🔄 **Smart caching** | Never wait for your data to process twice — cached results are automatically reused |
+| 📊 **Multi-framework interoperability** | Native conversion to/from NumPy, Pandas, Polars, Arrow, PyTorch, TensorFlow, JAX, and Spark |
+| 🏎️ **Multi-processing** | Fast parallel data processing with `map(num_proc=N)` |
+| 🔎 **Search & index** | Built-in FAISS and Elasticsearch index support for similarity search |
+| 📦 **JSON type** | Flexible JSON/structured data support with `Json()` feature type |
+
+# Installation
+
+## With pip
+
+🤗 Datasets can be installed from PyPi and should be installed in a virtual environment (venv or conda for instance):
+
+```bash
+pip install datasets
+```
+
+For the latest development version:
+
+```bash
+pip install "datasets @ git+https://github.com/huggingface/datasets.git"
+```
+
+## With conda
+
+```bash
+conda install -c huggingface -c conda-forge datasets
+```
+
+## Optional dependencies
+
+🤗 Datasets supports various optional features via extras:
+
+```bash
+# For audio (torchcodec)
+pip install datasets[audio]
+
+# For image/video (Pillow, torchcodec)
+pip install datasets[vision]
+
+# For PDFs/NIfTI (pdfplumber, nibabel)
+pip install datasets[pdfs,nibabel]
+
+# For PyTorch/TensorFlow/JAX integration
+pip install datasets[torch,tensorflow,jax]
+
+```
+
+For more details on installation, check the [installation page](https://huggingface.co/docs/datasets/installation).
+
+# Quick Start
+
+🤗 Datasets is made to be very simple to use — the API is centered around a single function, `datasets.load_dataset(dataset_name, **kwargs)`, that instantiates a dataset.
+
+Here is a quick example:
+
+```python
+from datasets import load_dataset
+
+# Load a dataset and print the first example in the training set
+squad_dataset = load_dataset('rajpurkar/squad')
+print(squad_dataset['train'][0])
+
+# Process the dataset - add a column with the length of the context texts
+dataset_with_length = squad_dataset.map(lambda x: {"length": len(x["context"])})
+
+# Tokenize the context texts (using a tokenizer from the 🤗 Transformers library)
+from transformers import AutoTokenizer
+tokenizer = AutoTokenizer.from_pretrained('bert-base-cased')
+
+tokenized_dataset = squad_dataset.map(lambda x: tokenizer(x['context']), batched=True)
+
+# Tokenize chat conversations with a chat template (using a model that supports chat templates)
+# This is useful for fine-tuning instruction/chat models
+
+# Load a popular chat dataset (ultrachat_200k contains ~200k AI assistant conversations)
+chat_dataset = load_dataset('HuggingFaceH4/ultrachat_200k', split='train_sft')
+
+chat_tokenizer = AutoTokenizer.from_pretrained('Qwen/Qwen2.5-7B-Instruct')
+
+def tokenize_chat(examples):
+ # Apply the chat template and tokenize in one step
+ return chat_tokenizer.apply_chat_template(examples["messages"])
+
+tokenized_chat_dataset = chat_dataset.map(tokenize_chat, batched=True)
+```
+
+## Streaming mode
+
+If your dataset is bigger than your disk or if you don't want to wait to download the data, you can use streaming:
+
+```python
+# Stream the dataset without downloading anything
+image_dataset = load_dataset('timm/imagenet-1k-wds', streaming=True)
+for example in image_dataset["train"]:
+ print(example["image"])
+ break
+```
+
+## Multi-modal data
+
+🤗 Datasets supports a wide variety of data types out of the box:
+
+```python
+# Audio dataset
+dataset = load_dataset("openslr/librispeech_asr", "clean")
+
+# Image dataset
+dataset = load_dataset("ILSVRC/imagenet-1k")
+
+# Video dataset
+dataset = load_dataset("Shofo/shofo-tiktok-general-small")
+
+# PDF documents
+dataset = load_dataset("pixparse/pdfa-eng-wds")
+
+# NIfTI (3D medical imaging)
+dataset = load_dataset("dartbrains/localizer", "betas")
+```
+
+## From local files
+
+```python
+# Load from local CSV
+dataset = load_dataset('csv', data_files='my_data.csv')
+
+# Load from local Parquet
+dataset = load_dataset('parquet', data_files='data/*.parquet')
+
+# Load from a local directory (auto-detect format)
+dataset = load_dataset('./path/to/data')
+```
+
+## From Python objects
+
+```python
+from datasets import Dataset
+
+# From a dictionary
+dataset = Dataset.from_dict({"text": ["Hello world", "How are you?"]})
+
+# From a list
+dataset = Dataset.from_list([{"text": "Hello world"}, {"text": "How are you?"}])
+
+# From Pandas
+import pandas as pd
+df = pd.DataFrame({"col1": [1, 2, 3], "col2": ["a", "b", "c"]})
+dataset = Dataset.from_pandas(df)
+
+# From a generator
+def gen():
+ for i in range(10):
+ yield {"value": i}
+dataset = Dataset.from_generator(gen)
+```
+
+For more details on using the library, check the [quick start guide](https://huggingface.co/docs/datasets/quickstart) and the specific pages on:
+
+- [Loading a dataset](https://huggingface.co/docs/datasets/loading)
+- [What's in a Dataset](https://huggingface.co/docs/datasets/access)
+- [Processing data with 🤗 Datasets](https://huggingface.co/docs/datasets/process)
+ - [Processing audio data](https://huggingface.co/docs/datasets/audio_process)
+ - [Processing image data](https://huggingface.co/docs/datasets/image_process)
+ - [Processing text data](https://huggingface.co/docs/datasets/nlp_process)
+ - [Processing PDF data](https://huggingface.co/docs/datasets/pdf_process)
+ - [Processing video data](https://huggingface.co/docs/datasets/video_process)
+- [Streaming a dataset](https://huggingface.co/docs/datasets/stream)
+
+# Core Classes
+
+The library provides two main dataset classes:
+
+| Class | Description |
+|-------|-------------|
+| `Dataset` | In-memory / memory-mapped dataset backed by Apache Arrow. Supports indexing, slicing, random access and caching. |
+| `IterableDataset` | Lazy, streamable dataset for large-scale / out-of-core processing. Supports streaming and infinite iteration. |
+
+Both are wrapped in `DatasetDict` / `IterableDatasetDict` for multi-split datasets (e.g., train/test/val).
+
+# Add a new dataset to the Hub
+
+We have a very detailed step-by-step guide to add a new dataset to the  datasets already provided on the [HuggingFace Datasets Hub](https://huggingface.co/datasets).
+
+You can find:
+- [how to upload a dataset to the Hub using your web browser or Python](https://huggingface.co/docs/datasets/upload_dataset) and also
+- [how to upload it using Git](https://huggingface.co/docs/datasets/share).
+
+# Disclaimers
+
+You can use 🤗 Datasets to load datasets based on versioned git repositories maintained by the dataset authors. For reproducibility reasons, we ask users to pin the `revision` of the repositories they use.
+
+If you're a dataset owner and wish to update any part of it (description, citation, license, etc.), or do not want your dataset to be included in the Hugging Face Hub, please get in touch by opening a discussion or a pull request in the Community tab of the dataset page. Thanks for your contribution to the ML community!
+
+# Contributing
+
+We welcome contributions! Please see our [Contributing Guide](CONTRIBUTING.md) for details on:
+
+- How to submit issues and pull requests
+- Code style guidelines (we use [Ruff](https://docs.astral.sh/ruff/))
+- Testing requirements
+- Documentation standards
+
+# BibTeX
+
+If you want to cite our 🤗 Datasets library, you can use our [paper](https://huggingface.co/papers/2109.02846):
+
+```bibtex
+@inproceedings{lhoest-etal-2021-datasets,
+ title = "Datasets: A Community Library for Natural Language Processing",
+ author = "Lhoest, Quentin and
+ Villanova del Moral, Albert and
+ Jernite, Yacine and
+ Thakur, Abhishek and
+ von Platen, Patrick and
+ Patil, Suraj and
+ Chaumond, Julien and
+ Drame, Mariama and
+ Plu, Julien and
+ Tunstall, Lewis and
+ Davison, Joe and
+ {\v{S}}a{\v{s}}ko, Mario and
+ Chhablani, Gunjan and
+ Malik, Bhavitvya and
+ Brandeis, Simon and
+ Le Scao, Teven and
+ Sanh, Victor and
+ Xu, Canwen and
+ Patry, Nicolas and
+ McMillan-Major, Angelina and
+ Schmid, Philipp and
+ Gugger, Sylvain and
+ Delangue, Cl{\'e}ment and
+ Matussi{\`e}re, Th{\'e}o and
+ Debut, Lysandre and
+ Bekman, Stas and
+ Cistac, Pierric and
+ Goehringer, Thibault and
+ Mustar, Victor and
+ Lagunas, Fran{\c{c}}ois and
+ Rush, Alexander and
+ Wolf, Thomas",
+ booktitle = "Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing: System Demonstrations",
+ month = nov,
+ year = "2021",
+ address = "Online and Punta Cana, Dominican Republic",
+ publisher = "Association for Computational Linguistics",
+ url = "https://aclanthology.org/2021.emnlp-demo.21",
+ pages = "175--184",
+ abstract = "The scale, variety, and quantity of publicly-available NLP datasets has grown rapidly as researchers propose new tasks, larger models, and novel benchmarks. Datasets is a community library for contemporary NLP designed to support this ecosystem. Datasets aims to standardize end-user interfaces, versioning, and documentation, while providing a lightweight front-end that behaves similarly for small datasets as for internet-scale corpora. The design of the library incorporates a distributed, community-driven approach to adding datasets and documenting usage. After a year of development, the library now includes more than 650 unique datasets, has more than 250 contributors, and has helped support a variety of novel cross-dataset research projects and shared tasks. The library is available at https://github.com/huggingface/datasets.",
+ eprint={2109.02846},
+ archivePrefix={arXiv},
+ primaryClass={cs.CL},
+}
+```
+
+If you need to cite a specific version of our 🤗 Datasets library for reproducibility, you can use the corresponding version Zenodo DOI from this [list](https://zenodo.org/search?q=conceptrecid:%224817768%22&sort=-version&all_versions=True).
diff --git a/venv/Lib/site-packages/datasets-5.0.0.dist-info/RECORD b/venv/Lib/site-packages/datasets-5.0.0.dist-info/RECORD
new file mode 100644
index 0000000000000000000000000000000000000000..ae85bd878ac92fd1cb55f9ced13783153ff1572b
--- /dev/null
+++ b/venv/Lib/site-packages/datasets-5.0.0.dist-info/RECORD
@@ -0,0 +1,291 @@
+../../Scripts/datasets-cli.exe,sha256=KFWQHy0Gnh0J6GqWrpeb1aex4skMTUzWUFLh5L75GF0,108406
+datasets-5.0.0.dist-info/AUTHORS,sha256=L0FBY23tCNHLmvsOKAbumHn8WZZIK98sH53JYxhAchU,327
+datasets-5.0.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+datasets-5.0.0.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+datasets-5.0.0.dist-info/METADATA,sha256=v2izydMJMyuRNfmA48q25NF19E2gJMOgu-cmYJiGx6A,23743
+datasets-5.0.0.dist-info/RECORD,,
+datasets-5.0.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets-5.0.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+datasets-5.0.0.dist-info/entry_points.txt,sha256=iM-h4A7OQCrZqr3L2mwiyMtPeFj8w4HAHzmI45y3tg0,69
+datasets-5.0.0.dist-info/top_level.txt,sha256=9A857YvCQm_Dg3UjeKkWPz9sDBos0t3zN2pf5krTemQ,9
+datasets/__init__.py,sha256=sRMzsbglG4fTNrVE32Yh49LJGbp5AMoXcX8lDQXVhXQ,1630
+datasets/__pycache__/__init__.cpython-311.pyc,,
+datasets/__pycache__/arrow_dataset.cpython-311.pyc,,
+datasets/__pycache__/arrow_reader.cpython-311.pyc,,
+datasets/__pycache__/arrow_writer.cpython-311.pyc,,
+datasets/__pycache__/builder.cpython-311.pyc,,
+datasets/__pycache__/combine.cpython-311.pyc,,
+datasets/__pycache__/config.cpython-311.pyc,,
+datasets/__pycache__/data_files.cpython-311.pyc,,
+datasets/__pycache__/dataset_dict.cpython-311.pyc,,
+datasets/__pycache__/distributed.cpython-311.pyc,,
+datasets/__pycache__/exceptions.cpython-311.pyc,,
+datasets/__pycache__/fingerprint.cpython-311.pyc,,
+datasets/__pycache__/hub.cpython-311.pyc,,
+datasets/__pycache__/info.cpython-311.pyc,,
+datasets/__pycache__/inspect.cpython-311.pyc,,
+datasets/__pycache__/iterable_dataset.cpython-311.pyc,,
+datasets/__pycache__/load.cpython-311.pyc,,
+datasets/__pycache__/naming.cpython-311.pyc,,
+datasets/__pycache__/search.cpython-311.pyc,,
+datasets/__pycache__/splits.cpython-311.pyc,,
+datasets/__pycache__/streaming.cpython-311.pyc,,
+datasets/__pycache__/table.cpython-311.pyc,,
+datasets/arrow_dataset.py,sha256=uy20R--VU08YaBmb1kRvFcwmR8nZVm11lOwgc7MhSqo,342300
+datasets/arrow_reader.py,sha256=54yLKdxCtZsuLgcZ3EiUvun_oors0zS3TTtQn7OPNa4,25215
+datasets/arrow_writer.py,sha256=30Ipiwwb-kiQskaYqh3rAUzG4VouzOW77AQ4Zc2Shy0,38706
+datasets/builder.py,sha256=0RdRBcf59t9sHZPyN_mMR_lrInlC6m8tC4Gd2lbeNJY,87946
+datasets/combine.py,sha256=MAqgg95h_DW1bb00gSZdD_a7DNnF9_kis8xwS4WFRKw,11761
+datasets/commands/__init__.py,sha256=rujbQtxJbwHhF9WQqp2DD9tfVTghDMJdl0v6H551Pcs,312
+datasets/commands/__pycache__/__init__.cpython-311.pyc,,
+datasets/commands/__pycache__/datasets_cli.cpython-311.pyc,,
+datasets/commands/__pycache__/delete_from_hub.cpython-311.pyc,,
+datasets/commands/__pycache__/env.cpython-311.pyc,,
+datasets/commands/__pycache__/test.cpython-311.pyc,,
+datasets/commands/datasets_cli.py,sha256=CRy2H60h2sxJDpfa4LZ4dxipdGIZlTL47pEXLV6tfwQ,1175
+datasets/commands/delete_from_hub.py,sha256=o0wdolb1r1Jnl6F0KdqKn3u0l8VR2od6KzbRoqrSNPM,1396
+datasets/commands/env.py,sha256=8qg-hpXSXXsHvtYFvJkn5rn9IncqPsjjx3nR8no4a2I,1239
+datasets/commands/test.py,sha256=ybafVFdQED7hc4JyY_U8BG9_CkkhL0Gp5dOOAuzaHUw,7824
+datasets/config.py,sha256=47pEpIB7t9D4JwicJ6uN3nhOS5n6X8_Ka1kgktisHZg,10590
+datasets/data_files.py,sha256=lsI_vKh_1BWJABYE5wRF3YUOlJQ4Fv77oDiYbyyvcuE,32812
+datasets/dataset_dict.py,sha256=3pbv445B79RP-_HiAPEDtZZTpdP8tbJHI1jKQotgJBw,128230
+datasets/distributed.py,sha256=YUEhqqx7uxCAQac9dZOHfJIMAVLPW5vDQkTqFZUbXRs,1815
+datasets/download/__init__.py,sha256=lbFOtITDaR7PHrhzJ8VfRnpaOT6NYozSxUcLv_GVfTg,281
+datasets/download/__pycache__/__init__.cpython-311.pyc,,
+datasets/download/__pycache__/download_config.cpython-311.pyc,,
+datasets/download/__pycache__/download_manager.cpython-311.pyc,,
+datasets/download/__pycache__/streaming_download_manager.cpython-311.pyc,,
+datasets/download/download_config.py,sha256=ODHFej2H-JIqFllklUFkiF4ILJ62l8clAGk13b8Ru3U,3796
+datasets/download/download_manager.py,sha256=Jq0BfYhQ7k5sT2tjbTh-zsj-b3MGS_IIP_bcjRUPfeo,12780
+datasets/download/streaming_download_manager.py,sha256=-eRyxNnKj-dH-3bzdKu-rFzHjhh57XKNwJc-087BNw0,7553
+datasets/exceptions.py,sha256=B93GwElhEvlhHPU9GBSY8if27jhRwu875-gL6B2CL6c,4185
+datasets/features/__init__.py,sha256=sfMwgbwdxKKrQTrM5SYQo20MirhHgSNaSb2ZAStmfQU,638
+datasets/features/__pycache__/__init__.cpython-311.pyc,,
+datasets/features/__pycache__/_torchcodec.cpython-311.pyc,,
+datasets/features/__pycache__/audio.cpython-311.pyc,,
+datasets/features/__pycache__/features.cpython-311.pyc,,
+datasets/features/__pycache__/image.cpython-311.pyc,,
+datasets/features/__pycache__/mesh.cpython-311.pyc,,
+datasets/features/__pycache__/nifti.cpython-311.pyc,,
+datasets/features/__pycache__/pdf.cpython-311.pyc,,
+datasets/features/__pycache__/translation.cpython-311.pyc,,
+datasets/features/__pycache__/video.cpython-311.pyc,,
+datasets/features/_torchcodec.py,sha256=Ws7JMYlUlPa7NHh1ZgxWQNrJV0c14kfYYCdrLjCkmjA,627
+datasets/features/audio.py,sha256=eQM6i6kO1057pe1Sob-0tY0GMF9Y8wULAXqsChABOzc,16110
+datasets/features/features.py,sha256=BMeibho5YIcW8WwIYyxqlYl21FciAVmQNRSOoBt9Fnw,98418
+datasets/features/image.py,sha256=0FMvaNyZjdGWCYf2QtY_kYUNqMEk8bTZMrhXcqmhLOc,18403
+datasets/features/mesh.py,sha256=nP6-yBamloz-S5pVPhq4sRSy-_8gwDQLKZ9xV6E7Xx8,12838
+datasets/features/nifti.py,sha256=Zu1fGqZPUK1qj-s5yG19cl0YwXM5QZQ_NDvHy61cFDA,14313
+datasets/features/pdf.py,sha256=Bjex_wBN3SWVSIf5E0ry2mHQL2yeLz5RyOrT_Ro0gVA,12207
+datasets/features/translation.py,sha256=aIJfNMXTTQLamEk4L8mfTDDdyzscZUnhSPAor8RjE_8,4490
+datasets/features/video.py,sha256=KWa61swxqNNtKLq9EC_KSP2VcMpGqCqMJjJHHHgWEYw,17261
+datasets/filesystems/__init__.py,sha256=jBDUQosQqEFIXUDLZwRWaTgNomwL6Fq2qiYPvvxuae0,1523
+datasets/filesystems/__pycache__/__init__.cpython-311.pyc,,
+datasets/filesystems/__pycache__/compression.cpython-311.pyc,,
+datasets/filesystems/compression.py,sha256=J16G_P0F9SxVfsLy9XLesyQJgvp4kpaO9MAihokEUvY,4528
+datasets/fingerprint.py,sha256=hnzIVs-jCCKJYqi6LvE8UWNmLNOJ97bvWz6uA1EAKiI,21922
+datasets/formatting/__init__.py,sha256=-pM10fSzw4MVj_L3NFWEv2sUyBh4mbnvCkfXgfS6WII,5412
+datasets/formatting/__pycache__/__init__.cpython-311.pyc,,
+datasets/formatting/__pycache__/formatting.cpython-311.pyc,,
+datasets/formatting/__pycache__/jax_formatter.cpython-311.pyc,,
+datasets/formatting/__pycache__/np_formatter.cpython-311.pyc,,
+datasets/formatting/__pycache__/polars_formatter.cpython-311.pyc,,
+datasets/formatting/__pycache__/tf_formatter.cpython-311.pyc,,
+datasets/formatting/__pycache__/torch_formatter.cpython-311.pyc,,
+datasets/formatting/formatting.py,sha256=8zMv7dfTyyEDvzNjKDzT7Eb2_fn2n6OknfGmYK_0-MM,26626
+datasets/formatting/jax_formatter.py,sha256=Hd2Lgl94bkka4LNz9_Q45VIKSV2KWkSLp3DRXy9laiY,7494
+datasets/formatting/np_formatter.py,sha256=TlXk6IlA_sxzeESg9jUUlhPX_npO6ImVj56A4Hp58Ms,5184
+datasets/formatting/polars_formatter.py,sha256=oTm4l30SgGha-Oku42C0dA91Y8f2oifF9aWvi3QITDk,4744
+datasets/formatting/tf_formatter.py,sha256=KRtG0yMzbVQ5TzA2WYa3WlZKblLojkqHzbNMKXY5geo,5318
+datasets/formatting/torch_formatter.py,sha256=N-10rmMTD8XVUaS6-LQ9xqVxfyAMVeSJEKS6cojy74k,5393
+datasets/hub.py,sha256=BuD2Jr4zMH8hpfbITMClQ3fq2TBteQQAvhcYiHHDwRg,5041
+datasets/info.py,sha256=R4AV3Of94kWMmFfZ7H66fusd7v2PWnRsCXpJaGRQFa4,19984
+datasets/inspect.py,sha256=dryCltaOJpfKyDxV2i-PWNE2P_VR_NPDBFRbLmeeQDQ,15498
+datasets/io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/io/__pycache__/__init__.cpython-311.pyc,,
+datasets/io/__pycache__/abc.cpython-311.pyc,,
+datasets/io/__pycache__/csv.cpython-311.pyc,,
+datasets/io/__pycache__/generator.cpython-311.pyc,,
+datasets/io/__pycache__/json.cpython-311.pyc,,
+datasets/io/__pycache__/parquet.cpython-311.pyc,,
+datasets/io/__pycache__/spark.cpython-311.pyc,,
+datasets/io/__pycache__/sql.cpython-311.pyc,,
+datasets/io/__pycache__/text.cpython-311.pyc,,
+datasets/io/abc.py,sha256=LwDMXYs6YkhZuz1JiMK4PDIqgNjv7I8xH3UMUELW2ys,1672
+datasets/io/csv.py,sha256=aNVSwbZVPyVvi3a5ZsJr11FMEOMKcXKZvoB1OcV0a2U,5198
+datasets/io/generator.py,sha256=WtUWhml4llyZd_6gPuOPcC_YiKc5xkTXAjTN-z4ivsY,2098
+datasets/io/json.py,sha256=pAKp3fhtGYOs2-6fqlTyqdKH_ehZztVnfvvdv1G_IaM,7010
+datasets/io/parquet.py,sha256=QzEK2oKCMopP3talCc5gCvP0xb2e34XTsDD9kF39s9c,5868
+datasets/io/spark.py,sha256=VUIODLHgIbiK0CI0UvthQ_gUO0MQDtHUozvw7Dfs8FI,1797
+datasets/io/sql.py,sha256=KZhCE2-al9QYz-e8rPctYvWQ07uRk11oS08OI1AtA-M,4175
+datasets/io/text.py,sha256=9BdYqlv-XXERc_WJwlD9GugwlAOv53iMVdrBoxKL2UI,1908
+datasets/iterable_dataset.py,sha256=8mkL1g_puZByq4pFp-FtQV6y0qBELP5wjAodXhZuzEE,237188
+datasets/load.py,sha256=LJcBF5_n1aNWg1EchitbtT627m4wTeM8zAmLx9QDuDA,79710
+datasets/naming.py,sha256=WzNdyHVGeAk_YPx3ZF09F-0g0WHVWil6UOQiKZFEFa0,3028
+datasets/packaged_modules/__init__.py,sha256=75vaQxbm5dRpUWRwOvXR2Iu2YXBcR5v1qb2K1tmf15A,7695
+datasets/packaged_modules/__pycache__/__init__.cpython-311.pyc,,
+datasets/packaged_modules/arrow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/packaged_modules/arrow/__pycache__/__init__.cpython-311.pyc,,
+datasets/packaged_modules/arrow/__pycache__/arrow.cpython-311.pyc,,
+datasets/packaged_modules/arrow/arrow.py,sha256=L2v-iUt583lAGJ9c_bzPm3gVqX0fV_LtoqSS73sRXUQ,3213
+datasets/packaged_modules/audiofolder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/packaged_modules/audiofolder/__pycache__/__init__.cpython-311.pyc,,
+datasets/packaged_modules/audiofolder/__pycache__/audiofolder.cpython-311.pyc,,
+datasets/packaged_modules/audiofolder/audiofolder.py,sha256=N4mOZypp8oTI-9FBSeEFE-oQ23U6ZmqPlFcqbUkviA8,1744
+datasets/packaged_modules/cache/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/packaged_modules/cache/__pycache__/__init__.cpython-311.pyc,,
+datasets/packaged_modules/cache/__pycache__/cache.cpython-311.pyc,,
+datasets/packaged_modules/cache/cache.py,sha256=qfhDDDuLNTl870-uRFwc6VTxJf9Icg5hYzpsIVrOf_M,8293
+datasets/packaged_modules/conll/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/packaged_modules/conll/__pycache__/__init__.cpython-311.pyc,,
+datasets/packaged_modules/conll/__pycache__/conll.cpython-311.pyc,,
+datasets/packaged_modules/conll/conll.py,sha256=wmajGnrB63PU_WosBgedZe7-nvfPraqqdj_VdFCm3P0,6846
+datasets/packaged_modules/csv/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/packaged_modules/csv/__pycache__/__init__.cpython-311.pyc,,
+datasets/packaged_modules/csv/__pycache__/csv.cpython-311.pyc,,
+datasets/packaged_modules/csv/csv.py,sha256=IY7nnFlOZIBmpcbdJ6wPPUVlhjd4K8rbw17h5YJU3Yw,8977
+datasets/packaged_modules/eval/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/packaged_modules/eval/__pycache__/__init__.cpython-311.pyc,,
+datasets/packaged_modules/eval/__pycache__/eval.cpython-311.pyc,,
+datasets/packaged_modules/eval/eval.py,sha256=GRFbIzWPzDCmWxWBefQ-kN-jjvxM0-2TdEZgVw-Qo0c,3226
+datasets/packaged_modules/folder_based_builder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/packaged_modules/folder_based_builder/__pycache__/__init__.cpython-311.pyc,,
+datasets/packaged_modules/folder_based_builder/__pycache__/folder_based_builder.cpython-311.pyc,,
+datasets/packaged_modules/folder_based_builder/folder_based_builder.py,sha256=YaQaj9fCEutUfrZINhuit0j6Szqaj6fOoUm6G9Wio9I,22939
+datasets/packaged_modules/generator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/packaged_modules/generator/__pycache__/__init__.cpython-311.pyc,,
+datasets/packaged_modules/generator/__pycache__/generator.cpython-311.pyc,,
+datasets/packaged_modules/generator/generator.py,sha256=HTpaY75-NypiL5g0mCgDcnvTaWvzdTPLIKnsFlLlpuI,1380
+datasets/packaged_modules/hdf5/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/packaged_modules/hdf5/__pycache__/__init__.cpython-311.pyc,,
+datasets/packaged_modules/hdf5/__pycache__/hdf5.cpython-311.pyc,,
+datasets/packaged_modules/hdf5/hdf5.py,sha256=CchNe8i3boLosSTddnmkn_AxATEEv-sbkDqhS1qwYaw,12592
+datasets/packaged_modules/iceberg/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/packaged_modules/iceberg/__pycache__/__init__.cpython-311.pyc,,
+datasets/packaged_modules/iceberg/__pycache__/iceberg.cpython-311.pyc,,
+datasets/packaged_modules/iceberg/iceberg.py,sha256=NeOVI_UWwJLQbOuznmvjlB3AnQN3gZ2eTFEOq5KlVz4,7148
+datasets/packaged_modules/imagefolder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/packaged_modules/imagefolder/__pycache__/__init__.cpython-311.pyc,,
+datasets/packaged_modules/imagefolder/__pycache__/imagefolder.cpython-311.pyc,,
+datasets/packaged_modules/imagefolder/imagefolder.py,sha256=UpMVe8TUyayzHsVSfKN5wiXcc94QdamMvxauI4oFdw4,1956
+datasets/packaged_modules/json/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/packaged_modules/json/__pycache__/__init__.cpython-311.pyc,,
+datasets/packaged_modules/json/__pycache__/json.cpython-311.pyc,,
+datasets/packaged_modules/json/json.py,sha256=l5R7OSnd2-1zyI0tRGOY3kLs_x02yA_fWwp4i8tdtgU,23343
+datasets/packaged_modules/lance/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/packaged_modules/lance/__pycache__/__init__.cpython-311.pyc,,
+datasets/packaged_modules/lance/__pycache__/lance.cpython-311.pyc,,
+datasets/packaged_modules/lance/lance.py,sha256=TQGbGU3_JPrV2C-fUglzLS2oqPe5Dyp7ALKniRISKjw,9880
+datasets/packaged_modules/meshfolder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/packaged_modules/meshfolder/__pycache__/__init__.cpython-311.pyc,,
+datasets/packaged_modules/meshfolder/__pycache__/meshfolder.cpython-311.pyc,,
+datasets/packaged_modules/meshfolder/meshfolder.py,sha256=J5Bha52ZP5NAGeWvkQmsNZnZVbxCbWqt3UEVNPDkrww,700
+datasets/packaged_modules/niftifolder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/packaged_modules/niftifolder/__pycache__/__init__.cpython-311.pyc,,
+datasets/packaged_modules/niftifolder/__pycache__/niftifolder.cpython-311.pyc,,
+datasets/packaged_modules/niftifolder/niftifolder.py,sha256=VPmkFLF590iCsG6b4vS3cONleyYM26cwUSi7kzj1ebg,575
+datasets/packaged_modules/pandas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/packaged_modules/pandas/__pycache__/__init__.cpython-311.pyc,,
+datasets/packaged_modules/pandas/__pycache__/pandas.cpython-311.pyc,,
+datasets/packaged_modules/pandas/pandas.py,sha256=qLtTReWnf2am9BZUa76sJwKags3Zg3qQsLoszZZq81E,1954
+datasets/packaged_modules/parquet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/packaged_modules/parquet/__pycache__/__init__.cpython-311.pyc,,
+datasets/packaged_modules/parquet/__pycache__/parquet.cpython-311.pyc,,
+datasets/packaged_modules/parquet/parquet.py,sha256=o90ueHItfJvlXJFBgwcaN_TCYwIZuayfD-FaFcxYzx4,11178
+datasets/packaged_modules/pdffolder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/packaged_modules/pdffolder/__pycache__/__init__.cpython-311.pyc,,
+datasets/packaged_modules/pdffolder/__pycache__/pdffolder.cpython-311.pyc,,
+datasets/packaged_modules/pdffolder/pdffolder.py,sha256=bPYBh9-XOr2C-gg_Fl8h-UKhsVQ7VXjBL2FfW8abiGU,565
+datasets/packaged_modules/spark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/packaged_modules/spark/__pycache__/__init__.cpython-311.pyc,,
+datasets/packaged_modules/spark/__pycache__/spark.cpython-311.pyc,,
+datasets/packaged_modules/spark/spark.py,sha256=T9LDFok-hAWDgcjyW0SIMfId5BXF3CAbvl9zFx8Ut6w,14685
+datasets/packaged_modules/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/packaged_modules/sql/__pycache__/__init__.cpython-311.pyc,,
+datasets/packaged_modules/sql/__pycache__/sql.cpython-311.pyc,,
+datasets/packaged_modules/sql/sql.py,sha256=DFGr-BdMyO6xE0MS2CCp-TtRZ-PUNKivUXSwF5R4K4g,4536
+datasets/packaged_modules/text/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/packaged_modules/text/__pycache__/__init__.cpython-311.pyc,,
+datasets/packaged_modules/text/__pycache__/text.cpython-311.pyc,,
+datasets/packaged_modules/text/text.py,sha256=TwrXqmv9-CsceZncffycwKi2BrdvzfLb4MKRQtFTuUM,6862
+datasets/packaged_modules/tsfile/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/packaged_modules/tsfile/__pycache__/__init__.cpython-311.pyc,,
+datasets/packaged_modules/tsfile/__pycache__/tsfile.cpython-311.pyc,,
+datasets/packaged_modules/tsfile/tsfile.py,sha256=AIfpPfmh0woeNh3IaoiImGPjekIHEFLHH_-YfuDF_wY,34665
+datasets/packaged_modules/videofolder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/packaged_modules/videofolder/__pycache__/__init__.cpython-311.pyc,,
+datasets/packaged_modules/videofolder/__pycache__/videofolder.cpython-311.pyc,,
+datasets/packaged_modules/videofolder/videofolder.py,sha256=HLTMldDZ3WfK8OAbI2wssBuNCP6ucRBpNLpCoJVDL10,807
+datasets/packaged_modules/webdataset/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/packaged_modules/webdataset/__pycache__/__init__.cpython-311.pyc,,
+datasets/packaged_modules/webdataset/__pycache__/_tenbin.cpython-311.pyc,,
+datasets/packaged_modules/webdataset/__pycache__/webdataset.cpython-311.pyc,,
+datasets/packaged_modules/webdataset/_tenbin.py,sha256=oovYsgR2R3eXSn1xSCLG3oTly1szKDP4UOiRp4ORdIk,8533
+datasets/packaged_modules/webdataset/webdataset.py,sha256=Bxd7J0DkxoMjZNWCU9l_X1qz5BDeNde_1Xl6o3rl38w,11166
+datasets/packaged_modules/xml/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/packaged_modules/xml/__pycache__/__init__.cpython-311.pyc,,
+datasets/packaged_modules/xml/__pycache__/xml.cpython-311.pyc,,
+datasets/packaged_modules/xml/xml.py,sha256=eMbwRIB1N7QSsvlIt1mPwseE_HYVuiXhyabxy-LILQk,2844
+datasets/parallel/__init__.py,sha256=wiRFK4x67ez2vvmjwM2Sb9R1yFdf38laSarU9y0Bido,76
+datasets/parallel/__pycache__/__init__.cpython-311.pyc,,
+datasets/parallel/__pycache__/parallel.cpython-311.pyc,,
+datasets/parallel/parallel.py,sha256=E-oOQ6zwKrkLFPwZ-3EOcr_aANJDhE-d6QTq7Mp7WvA,4738
+datasets/search.py,sha256=PxveDc1hy2Yux7Gtvat1_Qe_0D6aIKQBWI5KbV0gS6I,35592
+datasets/splits.py,sha256=10xtCAtS3XFHawm4AqZrfpXecfNcAcUnOJ9eVtwTkgY,23810
+datasets/streaming.py,sha256=7MIamZ2NmReUmJ_2pxgdSopIf7Oh5nFFAbb7WHLuW7E,5772
+datasets/table.py,sha256=p5nR05j8yRJkUM44RU5240xv1_2NHaSGVRz_m23LI3w,97900
+datasets/utils/__init__.py,sha256=PuZtB9YTbRyvdwubnsx-JGdHuMA7p0I0Rmh0E_uxYF0,999
+datasets/utils/__pycache__/__init__.cpython-311.pyc,,
+datasets/utils/__pycache__/_dataset_viewer.cpython-311.pyc,,
+datasets/utils/__pycache__/_dill.cpython-311.pyc,,
+datasets/utils/__pycache__/_filelock.cpython-311.pyc,,
+datasets/utils/__pycache__/deprecation_utils.cpython-311.pyc,,
+datasets/utils/__pycache__/doc_utils.cpython-311.pyc,,
+datasets/utils/__pycache__/experimental.cpython-311.pyc,,
+datasets/utils/__pycache__/extract.cpython-311.pyc,,
+datasets/utils/__pycache__/file_utils.cpython-311.pyc,,
+datasets/utils/__pycache__/filelock.cpython-311.pyc,,
+datasets/utils/__pycache__/hub.cpython-311.pyc,,
+datasets/utils/__pycache__/info_utils.cpython-311.pyc,,
+datasets/utils/__pycache__/json.cpython-311.pyc,,
+datasets/utils/__pycache__/logging.cpython-311.pyc,,
+datasets/utils/__pycache__/metadata.cpython-311.pyc,,
+datasets/utils/__pycache__/patching.cpython-311.pyc,,
+datasets/utils/__pycache__/py_utils.cpython-311.pyc,,
+datasets/utils/__pycache__/sharding.cpython-311.pyc,,
+datasets/utils/__pycache__/stratify.cpython-311.pyc,,
+datasets/utils/__pycache__/tf_utils.cpython-311.pyc,,
+datasets/utils/__pycache__/tqdm.cpython-311.pyc,,
+datasets/utils/__pycache__/track.cpython-311.pyc,,
+datasets/utils/__pycache__/typing.cpython-311.pyc,,
+datasets/utils/__pycache__/version.cpython-311.pyc,,
+datasets/utils/_dataset_viewer.py,sha256=SrE1N18S5yCoCx0rAhwaHNDVS9uhxjspA84iNT4TFRw,4397
+datasets/utils/_dill.py,sha256=RTXoz1e5nS-eHOsPvPhuKjIweEAl0cP8rFaKZkgmMQY,19409
+datasets/utils/_filelock.py,sha256=iXW3bxsIr5JWNemhKtF_-q_0ysajkUTItzMm8LY9LBY,2355
+datasets/utils/deprecation_utils.py,sha256=hTHwlzRs92NfNVudH71LMpW70sjbsP5amebrIgi3A-U,3452
+datasets/utils/doc_utils.py,sha256=HoSm0TFaQaCYGfDgNhpBJ4Xc2WQZuOD6dTxLd9D87fs,407
+datasets/utils/experimental.py,sha256=JgOjaEY3RWZ--3u0-ry82gLCDUpudfBfl-hWZ46SyS4,1097
+datasets/utils/extract.py,sha256=7rEeGuKxVrLlRwEkAeVFk-zMxq29XrIRyOvQihuEeHs,18318
+datasets/utils/file_utils.py,sha256=FpZJVIHXCXqJfiXXx9qYtxLMHajV3YgglhMUyi2F1I8,55868
+datasets/utils/filelock.py,sha256=H6C5dQGFCzVKyeDRRY8fZ4YGTEvvNd-MTjpL_sWYb5k,352
+datasets/utils/hub.py,sha256=V2JGolL5VjFT0YiEhI0sxJED_9tGdvma7lH22d64S9I,130
+datasets/utils/info_utils.py,sha256=SN9r8OSFSyGCNtpcpmCzbvSNU-_7LqTJsaRv07DeSYI,4217
+datasets/utils/json.py,sha256=Zy-XhSh8Tzz2XFUAfS6VwSdvi0e5C-T7jOjcd120H-0,5439
+datasets/utils/logging.py,sha256=c2g1gl3IV4C7A2-ky0yfrHZuYi4P41HRTdS3XCmdMew,5381
+datasets/utils/metadata.py,sha256=Hrmn8xUoEzwpJKG3Y6tfJt5t7nW1OCxNjfLTlEaxsrI,9367
+datasets/utils/patching.py,sha256=yrDATN-xaGgoM747brbx4ZDMnvp-teahfOaW53RNVyk,4967
+datasets/utils/py_utils.py,sha256=KGsYmXmOXD3QHRMd3aFRYXpj47biTSRwtzUU-d0lm8U,23246
+datasets/utils/resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datasets/utils/resources/__pycache__/__init__.cpython-311.pyc,,
+datasets/utils/resources/creators.json,sha256=XtIpMZefgBOdTevRrQTkFiufbgCbp_iyxseyphYQkn0,257
+datasets/utils/resources/languages.json,sha256=Z0rQNPsfje8zMi8KdvvwxF4APwwqcskJFUvhNiLAgPM,199138
+datasets/utils/resources/multilingualities.json,sha256=02Uc8RtRzfl13l98Y_alZm5HuMYwPzL78B0S5a1X-8c,205
+datasets/utils/resources/readme_structure.yaml,sha256=hNf9msoBZw5jfakQrDb0Af8T325TXdcaHsAO2MUcZvY,3877
+datasets/utils/resources/size_categories.json,sha256=_5nAP7z8R6t7_GfER81QudFO6Y1tqYu4AWrr4Aot8S8,171
+datasets/utils/sharding.py,sha256=VBQ4bRJQijMNDQTgFb1_ddlQ28wAcA0aQp4e-1jFIAk,4215
+datasets/utils/stratify.py,sha256=-MVaLmijYhGyKDpnZS9A8SiHekaIyVm84HVyIIQOmfg,4085
+datasets/utils/tf_utils.py,sha256=T3OysLGbkO7y-J-o9OVGyn9l-l-A3ruj-24JM_UULm8,24448
+datasets/utils/tqdm.py,sha256=IKCB-lz6BZwRokw1QHaQsWFGWL4ztWtRd5kCGzwp_Oc,4520
+datasets/utils/track.py,sha256=M81CGLn3MyJzHm98CQkbF3_1DG7evQsw-V52_Bp2paI,1838
+datasets/utils/typing.py,sha256=G11ytWmwjqVia2IdziRDIWvQ4mLJee-sKzgJfHqU16E,205
+datasets/utils/version.py,sha256=Z82cHpjTbQVJyWgnwSU8DsW2G0y-sSbSoOVeQrAds9k,3281
diff --git a/venv/Lib/site-packages/datasets-5.0.0.dist-info/REQUESTED b/venv/Lib/site-packages/datasets-5.0.0.dist-info/REQUESTED
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets-5.0.0.dist-info/WHEEL b/venv/Lib/site-packages/datasets-5.0.0.dist-info/WHEEL
new file mode 100644
index 0000000000000000000000000000000000000000..505164bc02d63fe6b0b3299f849a77c5f1beeb41
--- /dev/null
+++ b/venv/Lib/site-packages/datasets-5.0.0.dist-info/WHEEL
@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.8.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+
diff --git a/venv/Lib/site-packages/datasets-5.0.0.dist-info/entry_points.txt b/venv/Lib/site-packages/datasets-5.0.0.dist-info/entry_points.txt
new file mode 100644
index 0000000000000000000000000000000000000000..a93c17922e3b5496fe8e1a43f3defa58727f5220
--- /dev/null
+++ b/venv/Lib/site-packages/datasets-5.0.0.dist-info/entry_points.txt
@@ -0,0 +1,2 @@
+[console_scripts]
+datasets-cli = datasets.commands.datasets_cli:main
diff --git a/venv/Lib/site-packages/datasets-5.0.0.dist-info/top_level.txt b/venv/Lib/site-packages/datasets-5.0.0.dist-info/top_level.txt
new file mode 100644
index 0000000000000000000000000000000000000000..aee11b288aa3e6803c53bde002f7594c44497f5b
--- /dev/null
+++ b/venv/Lib/site-packages/datasets-5.0.0.dist-info/top_level.txt
@@ -0,0 +1 @@
+datasets
diff --git a/venv/Lib/site-packages/datasets/__pycache__/fingerprint.cpython-311.pyc b/venv/Lib/site-packages/datasets/__pycache__/fingerprint.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1e76f72ca5c86a8b9ccef7cecfaf5374795dbbb5
Binary files /dev/null and b/venv/Lib/site-packages/datasets/__pycache__/fingerprint.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/__pycache__/hub.cpython-311.pyc b/venv/Lib/site-packages/datasets/__pycache__/hub.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3a42b8363c779215ea4cbe978d5f6208928b9f98
Binary files /dev/null and b/venv/Lib/site-packages/datasets/__pycache__/hub.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/__pycache__/info.cpython-311.pyc b/venv/Lib/site-packages/datasets/__pycache__/info.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..640809f6256ec4cad8b7ed9d98bc7f261bb8dee4
Binary files /dev/null and b/venv/Lib/site-packages/datasets/__pycache__/info.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/__pycache__/inspect.cpython-311.pyc b/venv/Lib/site-packages/datasets/__pycache__/inspect.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..aa9c13b5877f4b83aaf635e5c8049e3d017c5ed9
Binary files /dev/null and b/venv/Lib/site-packages/datasets/__pycache__/inspect.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/__pycache__/iterable_dataset.cpython-311.pyc b/venv/Lib/site-packages/datasets/__pycache__/iterable_dataset.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..081069407cfd39721859243417f76ce9e16a3329
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/__pycache__/iterable_dataset.cpython-311.pyc
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f1069d468cb4557bca9e7fd248bf22f7bd9f9fd154bf8cb1d16b9387a338ab0
+size 294705
diff --git a/venv/Lib/site-packages/datasets/__pycache__/load.cpython-311.pyc b/venv/Lib/site-packages/datasets/__pycache__/load.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..fc2001eeefae59ad4944be469015c57b6f940692
Binary files /dev/null and b/venv/Lib/site-packages/datasets/__pycache__/load.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/__pycache__/naming.cpython-311.pyc b/venv/Lib/site-packages/datasets/__pycache__/naming.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3b3b18f5a93d4ff3a0306e45c6b9058bac78b3fd
Binary files /dev/null and b/venv/Lib/site-packages/datasets/__pycache__/naming.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/__pycache__/search.cpython-311.pyc b/venv/Lib/site-packages/datasets/__pycache__/search.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..729d0a08ae6761524af63a727084297e47027c44
Binary files /dev/null and b/venv/Lib/site-packages/datasets/__pycache__/search.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/__pycache__/splits.cpython-311.pyc b/venv/Lib/site-packages/datasets/__pycache__/splits.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..da5703eeef4c00848f9c9720266fb94c6429b4dd
Binary files /dev/null and b/venv/Lib/site-packages/datasets/__pycache__/splits.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/__pycache__/streaming.cpython-311.pyc b/venv/Lib/site-packages/datasets/__pycache__/streaming.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..fc8351ebd2e7e2cbf766844933d5a532debd7164
Binary files /dev/null and b/venv/Lib/site-packages/datasets/__pycache__/streaming.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/__pycache__/table.cpython-311.pyc b/venv/Lib/site-packages/datasets/__pycache__/table.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e95df41a66d4bf4f197b52c556f098320720c728
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/__pycache__/table.cpython-311.pyc
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd4777d458c7f0ec20c6059316202985d53515db31c4e22beb61c28f8e9c2f7b
+size 126990
diff --git a/venv/Lib/site-packages/datasets/arrow_dataset.py b/venv/Lib/site-packages/datasets/arrow_dataset.py
new file mode 100644
index 0000000000000000000000000000000000000000..59451a640e6d19d8649e9ccca6cdad375066df55
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/arrow_dataset.py
@@ -0,0 +1,7391 @@
+# Copyright 2020 The HuggingFace Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Simple Dataset wrapping an Arrow Table."""
+
+import asyncio
+import contextlib
+import copy
+import glob
+import inspect
+import itertools
+import json
+import math
+import os
+import posixpath
+import random
+import re
+import shutil
+import string
+import sys
+import tempfile
+import time
+import warnings
+import weakref
+from collections import Counter, defaultdict
+from collections.abc import Iterable, Iterator, Mapping
+from collections.abc import Sequence as Sequence_
+from copy import deepcopy
+from functools import partial, wraps
+from math import ceil, floor
+from pathlib import Path
+from random import sample
+from typing import (
+ TYPE_CHECKING,
+ Any,
+ BinaryIO,
+ Callable,
+ Optional,
+ Union,
+ overload,
+)
+
+import fsspec
+import multiprocess as mp
+import numpy as np
+import pandas as pd
+import pyarrow as pa
+import pyarrow.compute as pc
+import pyarrow.dataset as pds
+from fsspec.core import url_to_fs
+from fsspec.implementations.dirfs import DirFileSystem
+from huggingface_hub import (
+ CommitInfo,
+ CommitOperationAdd,
+ CommitOperationDelete,
+ DatasetCard,
+ DatasetCardData,
+ HfApi,
+ HfFileSystem,
+ HfFileSystemResolvedPath,
+)
+from huggingface_hub.utils import EntryNotFoundError, HfHubHTTPError, RepositoryNotFoundError
+from packaging import version
+from tqdm.contrib.concurrent import thread_map
+
+from . import __version__, config
+from .arrow_reader import ArrowReader
+from .arrow_writer import ArrowWriter, OptimizedTypedSequence
+from .data_files import sanitize_patterns
+from .download.streaming_download_manager import xgetsize
+from .features import Audio, ClassLabel, Features, Image, List, Value, Video
+from .features.features import (
+ FeatureType,
+ _align_features,
+ _check_if_features_can_be_aligned,
+ _fix_for_backward_compatible_features,
+ generate_from_arrow_type,
+ pandas_types_mapper,
+ require_decoding,
+)
+from .filesystems import is_remote_filesystem
+from .fingerprint import (
+ fingerprint_transform,
+ format_kwargs_for_fingerprint,
+ format_transform_for_fingerprint,
+ generate_fingerprint,
+ generate_random_fingerprint,
+ get_temporary_cache_files_directory,
+ is_caching_enabled,
+ maybe_register_dataset_for_temp_dir_deletion,
+ update_fingerprint,
+ validate_fingerprint,
+)
+from .formatting import format_table, get_format_type_from_alias, get_formatter, query_table
+from .formatting.formatting import LazyDict, _is_range_contiguous
+from .info import DatasetInfo, DatasetInfosDict
+from .naming import _split_re
+from .search import IndexableMixin
+from .splits import NamedSplit, Split, SplitDict, SplitInfo
+from .table import (
+ InMemoryTable,
+ MemoryMappedTable,
+ Table,
+ _batch_accumulate_arrow_table_by_columns,
+ _batch_arrow_table,
+ _memory_mapped_record_batch_reader_from_file,
+ cast_array_to_feature,
+ concat_tables,
+ embed_table_storage,
+ list_table_cache_files,
+ table_cast,
+ table_iter,
+ table_visitor,
+)
+from .utils import logging
+from .utils import tqdm as hf_tqdm
+from .utils.file_utils import estimate_dataset_size
+from .utils.info_utils import is_small_dataset
+from .utils.metadata import MetadataConfigs
+from .utils.py_utils import (
+ Literal,
+ asdict,
+ convert_file_size_to_int,
+ glob_pattern_to_regex,
+ iflatmap_unordered,
+ string_to_dict,
+)
+from .utils.stratify import stratified_shuffle_split_generate_indices
+from .utils.tf_utils import dataset_to_tf, minimal_tf_collate_fn, multiprocess_dataset_to_tf
+from .utils.typing import ListLike, PathLike
+
+
+if config.HF_HUB_VERSION >= version.parse("1.6.0"):
+ from huggingface_hub.errors import BucketNotFoundError
+ from huggingface_hub.hf_file_system import HfFileSystemResolvedBucketPath, HfFileSystemResolvedRepositoryPath
+
+else:
+ BucketNotFoundError = None
+ HfFileSystemResolvedBucketPath = None
+ HfFileSystemResolvedRepositoryPath = HfFileSystemResolvedPath
+
+
+if TYPE_CHECKING:
+ import sqlite3
+
+ import polars as pl
+ import pyspark
+ import sqlalchemy
+
+ from .dataset_dict import DatasetDict
+ from .iterable_dataset import IterableDataset
+
+logger = logging.get_logger(__name__)
+
+PUSH_TO_HUB_WITHOUT_METADATA_CONFIGS_SPLIT_PATTERN_SHARDED = (
+ "data/{split}-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.parquet"
+)
+
+
+class DatasetInfoMixin:
+ """This base class exposes some attributes of DatasetInfo
+ at the base level of the Dataset for easy access.
+ """
+
+ def __init__(self, info: DatasetInfo, split: Optional[NamedSplit]):
+ self._info = info
+ self._split = split
+
+ @property
+ def info(self):
+ """[`~datasets.DatasetInfo`] object containing all the metadata in the dataset."""
+ return self._info
+
+ @property
+ def split(self):
+ """[`~datasets.NamedSplit`] object corresponding to a named dataset split."""
+ return self._split
+
+ @property
+ def builder_name(self) -> str:
+ return self._info.builder_name
+
+ @property
+ def citation(self) -> str:
+ return self._info.citation
+
+ @property
+ def config_name(self) -> str:
+ return self._info.config_name
+
+ @property
+ def dataset_size(self) -> Optional[int]:
+ return self._info.dataset_size
+
+ @property
+ def description(self) -> str:
+ return self._info.description
+
+ @property
+ def download_checksums(self) -> Optional[dict]:
+ return self._info.download_checksums
+
+ @property
+ def download_size(self) -> Optional[int]:
+ return self._info.download_size
+
+ @property
+ def features(self) -> Optional[Features]:
+ return self._info.features.copy() if self._info.features is not None else None
+
+ @property
+ def homepage(self) -> Optional[str]:
+ return self._info.homepage
+
+ @property
+ def license(self) -> Optional[str]:
+ return self._info.license
+
+ @property
+ def size_in_bytes(self) -> Optional[int]:
+ return self._info.size_in_bytes
+
+ @property
+ def supervised_keys(self):
+ return self._info.supervised_keys
+
+ @property
+ def version(self):
+ return self._info.version
+
+
+class TensorflowDatasetMixin:
+ _TF_DATASET_REFS = set()
+
+ @staticmethod
+ def _get_output_signature(
+ dataset: "Dataset",
+ collate_fn: Callable,
+ collate_fn_args: dict,
+ cols_to_retain: Optional[list[str]] = None,
+ batch_size: Optional[int] = None,
+ num_test_batches: int = 20,
+ ):
+ """Private method used by `to_tf_dataset()` to find the shapes and dtypes of samples from this dataset
+ after being passed through the collate_fn. Tensorflow needs an exact signature for tf.numpy_function, so
+ the only way to do this is to run test batches - the collator may add or rename columns, so we can't figure
+ it out just by inspecting the dataset.
+
+ Args:
+ dataset (`Dataset`): Dataset to load samples from.
+ collate_fn(`bool`): Shuffle the dataset order when loading. Recommended True for training, False for
+ validation/evaluation.
+ collate_fn(`Callable`): A function or callable object (such as a `DataCollator`) that will collate
+ lists of samples into a batch.
+ collate_fn_args (`Dict`): A `dict` of keyword arguments to be passed to the
+ `collate_fn`.
+ batch_size (`int`, optional): The size of batches loaded from the dataset. Used for shape inference.
+ Can be None, which indicates that batch sizes can be variable.
+ num_test_batches (`int`): The number of batches to load from the dataset for shape inference.
+
+ Returns:
+ `dict`: Dict mapping column names to tf.Tensorspec objects
+ `dict`: Dict mapping column names to np.dtype objects
+ """
+ if config.TF_AVAILABLE:
+ import tensorflow as tf
+ else:
+ raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.")
+
+ if len(dataset) == 0:
+ raise ValueError("Unable to get the output signature because the dataset is empty.")
+ if batch_size is not None:
+ batch_size = min(len(dataset), batch_size)
+ test_batch_size = 1
+
+ if cols_to_retain is not None:
+ cols_to_retain = list(set(cols_to_retain + ["label_ids", "label", "labels"]))
+
+ test_batches = []
+ for _ in range(num_test_batches):
+ indices = sample(range(len(dataset)), test_batch_size)
+ test_batch = dataset[indices]
+ if cols_to_retain is not None:
+ test_batch = {key: value for key, value in test_batch.items() if key in cols_to_retain}
+ test_batch = [{key: value[i] for key, value in test_batch.items()} for i in range(test_batch_size)]
+ test_batch = collate_fn(test_batch, **collate_fn_args)
+ test_batches.append(test_batch)
+
+ tf_columns_to_signatures = {}
+ np_columns_to_dtypes = {}
+ for column in test_batches[0].keys():
+ raw_arrays = [batch[column] for batch in test_batches]
+ # In case the collate_fn returns something strange
+ np_arrays = []
+ for array in raw_arrays:
+ if isinstance(array, np.ndarray):
+ np_arrays.append(array)
+ elif isinstance(array, tf.Tensor):
+ np_arrays.append(array.numpy())
+ else:
+ np_arrays.append(np.array(array))
+
+ if np.issubdtype(np_arrays[0].dtype, np.integer) or np_arrays[0].dtype == bool:
+ tf_dtype = tf.int64
+ np_dtype = np.int64
+ elif np.issubdtype(np_arrays[0].dtype, np.number):
+ tf_dtype = tf.float32
+ np_dtype = np.float32
+ elif np_arrays[0].dtype.kind == "U": # Unicode strings
+ np_dtype = np.str_
+ tf_dtype = tf.string
+ else:
+ raise RuntimeError(
+ f"Unrecognized array dtype {np_arrays[0].dtype}. \n"
+ "Nested types and image/audio types are not supported yet."
+ )
+ shapes = [array.shape for array in np_arrays]
+ static_shape = []
+ for dim in range(len(shapes[0])):
+ sizes = {shape[dim] for shape in shapes}
+ if dim == 0:
+ static_shape.append(batch_size)
+ continue
+ if len(sizes) == 1: # This dimension looks constant
+ static_shape.append(sizes.pop())
+ else: # Use None for variable dimensions
+ static_shape.append(None)
+ tf_columns_to_signatures[column] = tf.TensorSpec(shape=static_shape, dtype=tf_dtype)
+ np_columns_to_dtypes[column] = np_dtype
+
+ return tf_columns_to_signatures, np_columns_to_dtypes
+
+ def to_tf_dataset(
+ self,
+ batch_size: Optional[int] = None,
+ columns: Optional[Union[str, list[str]]] = None,
+ shuffle: bool = False,
+ collate_fn: Optional[Callable] = None,
+ drop_remainder: bool = False,
+ collate_fn_args: Optional[dict[str, Any]] = None,
+ label_cols: Optional[Union[str, list[str]]] = None,
+ prefetch: bool = True,
+ num_workers: int = 0,
+ num_test_batches: int = 20,
+ ):
+ """Create a `tf.data.Dataset` from the underlying Dataset. This `tf.data.Dataset` will load and collate batches from
+ the Dataset, and is suitable for passing to methods like `model.fit()` or `model.predict()`. The dataset will yield
+ `dicts` for both inputs and labels unless the `dict` would contain only a single key, in which case a raw
+ `tf.Tensor` is yielded instead.
+
+ Args:
+ batch_size (`int`, *optional*):
+ Size of batches to load from the dataset. Defaults to `None`, which implies that the dataset won't be
+ batched, but the returned dataset can be batched later with `tf_dataset.batch(batch_size)`.
+ columns (`List[str]` or `str`, *optional*):
+ Dataset column(s) to load in the `tf.data.Dataset`.
+ Column names that are created by the `collate_fn` and that do not exist in the original dataset can be used.
+ shuffle(`bool`, defaults to `False`):
+ Shuffle the dataset order when loading. Recommended `True` for training, `False` for
+ validation/evaluation.
+ drop_remainder(`bool`, defaults to `False`):
+ Drop the last incomplete batch when loading. Ensures
+ that all batches yielded by the dataset will have the same length on the batch dimension.
+ collate_fn(`Callable`, *optional*):
+ A function or callable object (such as a `DataCollator`) that will collate
+ lists of samples into a batch.
+ collate_fn_args (`Dict`, *optional*):
+ An optional `dict` of keyword arguments to be passed to the
+ `collate_fn`.
+ label_cols (`List[str]` or `str`, defaults to `None`):
+ Dataset column(s) to load as labels.
+ Note that many models compute loss internally rather than letting Keras do it, in which case
+ passing the labels here is optional, as long as they're in the input `columns`.
+ prefetch (`bool`, defaults to `True`):
+ Whether to run the dataloader in a separate thread and maintain
+ a small buffer of batches for training. Improves performance by allowing data to be loaded in the
+ background while the model is training.
+ num_workers (`int`, defaults to `0`):
+ Number of workers to use for loading the dataset.
+ num_test_batches (`int`, defaults to `20`):
+ Number of batches to use to infer the output signature of the dataset.
+ The higher this number, the more accurate the signature will be, but the longer it will take to
+ create the dataset.
+
+ Returns:
+ `tf.data.Dataset`
+
+ Example:
+
+ ```py
+ >>> ds_train = ds["train"].to_tf_dataset(
+ ... columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'],
+ ... shuffle=True,
+ ... batch_size=16,
+ ... collate_fn=data_collator,
+ ... )
+ ```
+ """
+ if config.TF_AVAILABLE:
+ import tensorflow as tf
+ else:
+ raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.")
+
+ if (isinstance(columns, list) and len(columns) == 1) or (
+ isinstance(label_cols, list) and len(label_cols) == 1
+ ):
+ warnings.warn(
+ "The output of `to_tf_dataset` will change when a passing single element list for `labels` or "
+ "`columns` in the next datasets version. To return a tuple structure rather than dict, pass a "
+ "single string.\n"
+ "Old behaviour: columns=['a'], labels=['labels'] -> (tf.Tensor, tf.Tensor) \n"
+ " : columns='a', labels='labels' -> (tf.Tensor, tf.Tensor) \n"
+ "New behaviour: columns=['a'],labels=['labels'] -> ({'a': tf.Tensor}, {'labels': tf.Tensor}) \n"
+ " : columns='a', labels='labels' -> (tf.Tensor, tf.Tensor) ",
+ FutureWarning,
+ )
+
+ if isinstance(tf.distribute.get_strategy(), tf.distribute.TPUStrategy):
+ logger.warning(
+ "Note that to_tf_dataset() loads the data with a generator rather than a full tf.data "
+ "pipeline and is not compatible with remote TPU connections. If you encounter errors, please "
+ "try using a TPU VM or, if your data can fit in memory, loading it into memory as a dict of "
+ "Tensors instead of streaming with to_tf_dataset()."
+ )
+
+ if collate_fn is None:
+ # Set a very simple default collator that just stacks things together
+ collate_fn = minimal_tf_collate_fn
+ if collate_fn_args is None:
+ collate_fn_args = {}
+ if label_cols and not columns:
+ raise ValueError("Cannot specify label_cols without specifying columns!")
+ if label_cols is None:
+ label_cols = []
+ elif isinstance(label_cols, str):
+ label_cols = [label_cols]
+ if len(set(label_cols)) < len(label_cols):
+ raise ValueError("List of label_cols contains duplicates.")
+ if columns:
+ if isinstance(columns, str):
+ columns = [columns]
+ if len(set(columns)) < len(columns):
+ raise ValueError("List of columns contains duplicates.")
+ cols_to_retain = list(set(columns + label_cols))
+ else:
+ cols_to_retain = None # Indicates keeping all valid columns
+ columns = []
+
+ if self.format["type"] not in ["custom", "numpy"]:
+ dataset = self.with_format("numpy")
+ else:
+ dataset = self
+
+ # TODO(Matt, QL): deprecate the retention of label_ids and label
+
+ output_signature, columns_to_np_types = dataset._get_output_signature(
+ dataset,
+ collate_fn=collate_fn,
+ collate_fn_args=collate_fn_args,
+ cols_to_retain=cols_to_retain,
+ batch_size=batch_size if drop_remainder else None,
+ num_test_batches=num_test_batches,
+ )
+
+ if "labels" in output_signature:
+ if ("label_ids" in columns or "label" in columns) and "labels" not in columns:
+ columns = [col for col in columns if col not in ["label_ids", "label"]] + ["labels"]
+ if ("label_ids" in label_cols or "label" in label_cols) and "labels" not in label_cols:
+ label_cols = [col for col in label_cols if col not in ["label_ids", "label"]] + ["labels"]
+
+ for col in columns:
+ if col not in output_signature:
+ raise ValueError(f"Column {col} not found in dataset!")
+
+ for col in label_cols:
+ if col not in output_signature:
+ raise ValueError(f"Label column {col} not found in dataset!")
+
+ if num_workers == 0:
+ tf_dataset = dataset_to_tf(
+ dataset=dataset,
+ cols_to_retain=cols_to_retain,
+ collate_fn=collate_fn,
+ collate_fn_args=collate_fn_args,
+ columns_to_np_types=columns_to_np_types,
+ output_signature=output_signature,
+ shuffle=shuffle,
+ batch_size=batch_size,
+ drop_remainder=drop_remainder,
+ )
+ elif num_workers > 0:
+ if batch_size is None:
+ raise NotImplementedError(
+ "`batch_size` must be specified when using multiple workers, as unbatched multiprocessing "
+ "is not supported yet. Please provide a `batch_size` if `num_workers` is greater than 0."
+ )
+ tf_dataset = multiprocess_dataset_to_tf(
+ dataset=dataset,
+ cols_to_retain=cols_to_retain,
+ collate_fn=collate_fn,
+ collate_fn_args=collate_fn_args,
+ columns_to_np_types=columns_to_np_types,
+ output_signature=output_signature,
+ shuffle=shuffle,
+ batch_size=batch_size,
+ drop_remainder=drop_remainder,
+ num_workers=num_workers,
+ )
+ else:
+ raise ValueError("num_workers must be >= 0")
+
+ def split_features_and_labels(input_batch):
+ # TODO(Matt, QL): deprecate returning the dict content when there's only one key
+ features = {key: tensor for key, tensor in input_batch.items() if key in columns}
+ labels = {key: tensor for key, tensor in input_batch.items() if key in label_cols}
+ if len(features) == 1:
+ features = list(features.values())[0]
+ if len(labels) == 1:
+ labels = list(labels.values())[0]
+ if isinstance(labels, dict) and len(labels) == 0:
+ return features
+ else:
+ return features, labels
+
+ if cols_to_retain is not None:
+ tf_dataset = tf_dataset.map(split_features_and_labels)
+
+ if prefetch:
+ tf_dataset = tf_dataset.prefetch(tf.data.experimental.AUTOTUNE)
+
+ # Remove a reference to the open Arrow file on delete
+ def cleanup_callback(ref):
+ dataset.__del__()
+ self._TF_DATASET_REFS.remove(ref)
+
+ self._TF_DATASET_REFS.add(weakref.ref(tf_dataset, cleanup_callback))
+
+ return tf_dataset
+
+
+class DatasetTransformationNotAllowedError(Exception):
+ pass
+
+
+def transmit_format(func):
+ """Wrapper for dataset transforms that recreate a new Dataset to transmit the format of the original dataset to the new dataset"""
+
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ if args:
+ self: "Dataset" = args[0]
+ args = args[1:]
+ else:
+ self: "Dataset" = kwargs.pop("self")
+ # don't use self.format since it returns a list of columns for 'columns' even if self_format_columns is None
+ unformatted_columns = set(self.column_names) - set(self._format_columns or [])
+ self_format = {
+ "type": self._format_type,
+ "format_kwargs": self._format_kwargs,
+ "columns": self._format_columns,
+ "output_all_columns": self._output_all_columns,
+ }
+ # apply actual function
+ out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs)
+ datasets: list["Dataset"] = list(out.values()) if isinstance(out, dict) else [out]
+ # re-apply format to the output
+ for dataset in datasets:
+ new_format = self_format.copy()
+ if new_format["columns"] is not None: # new formatted columns = (columns - previously unformatted columns)
+ # sort the columns to have a deterministic list of columns that we can compare with `out_format`
+ new_format["columns"] = sorted(set(dataset.column_names) - unformatted_columns)
+ out_format = {
+ "type": dataset._format_type,
+ "format_kwargs": dataset._format_kwargs,
+ "columns": sorted(dataset._format_columns) if dataset._format_columns is not None else None,
+ "output_all_columns": dataset._output_all_columns,
+ }
+ if out_format != new_format:
+ fingerprint = dataset._fingerprint
+ dataset.set_format(**new_format)
+ dataset._fingerprint = fingerprint
+ return out
+
+ wrapper._decorator_name_ = "transmit_format"
+ return wrapper
+
+
+def update_metadata_with_features(table: Table, features: Features):
+ """To be used in dataset transforms that modify the features of the dataset, in order to update the features stored in the metadata of its schema."""
+ features = Features({col_name: features[col_name] for col_name in table.column_names})
+ if table.schema.metadata is None or b"huggingface" not in table.schema.metadata:
+ pa_metadata = ArrowWriter._build_metadata(DatasetInfo(features=features))
+ else:
+ metadata = json.loads(table.schema.metadata[b"huggingface"].decode())
+ if "info" not in metadata:
+ metadata["info"] = asdict(DatasetInfo(features=features))
+ else:
+ metadata["info"]["features"] = asdict(DatasetInfo(features=features))["features"]
+ pa_metadata = {"huggingface": json.dumps(metadata)}
+ table = table.replace_schema_metadata(pa_metadata)
+ return table
+
+
+def _check_table(table) -> Table:
+ """We check the table type to make sure it's an instance of :class:`datasets.table.Table`"""
+ if isinstance(table, pa.Table):
+ # for a pyarrow table, we can just consider it as a in-memory table
+ # this is here for backward compatibility
+ return InMemoryTable(table)
+ elif isinstance(table, Table):
+ return table
+ else:
+ raise TypeError(f"Expected a pyarrow.Table or a datasets.table.Table object, but got {table}.")
+
+
+def _check_column_names(column_names: list[str]):
+ """Check the column names to make sure they don't contain duplicates."""
+ counter = Counter(column_names)
+ if not all(count == 1 for count in counter.values()):
+ duplicated_columns = [col for col in counter if counter[col] > 1]
+ raise ValueError(f"The table can't have duplicated columns but columns {duplicated_columns} are duplicated.")
+
+
+def _check_valid_indices_value(index, size):
+ if (index < 0 and index + size < 0) or (index >= size):
+ raise IndexError(f"Index {index} out of range for dataset of size {size}.")
+
+
+class NonExistentDatasetError(Exception):
+ """Used when we expect the existence of a dataset"""
+
+ pass
+
+
+class Column(Sequence_):
+ """
+ An iterable for a specific column of a [`Dataset`].
+
+ Example:
+
+ Iterate on the texts of the "text" column of a dataset:
+
+ ```python
+ for text in dataset["text"]:
+ ...
+ ```
+
+ It also works with nested columns:
+
+ ```python
+ for source in dataset["metadata"]["source"]:
+ ...
+ ```
+ """
+
+ def __init__(self, source: Union["Dataset", "Column"], column_name: str):
+ self.source = source
+ self.column_name = column_name
+ if not isinstance(source.features, dict) or column_name not in source.features:
+ raise ValueError(f"Column '{column_name}' doesn't exist.")
+ self.features = source.features[column_name]
+
+ def __iter__(self) -> Iterator[Any]:
+ if isinstance(self.source, Dataset):
+ if self.source._format_type == "custom":
+ # the formatting transform may require all columns
+ source = self.source
+ else:
+ source = self.source._fast_select_column(self.column_name)
+ else:
+ source = self.source
+ for example in source:
+ yield example[self.column_name]
+
+ def __getitem__(self, key: Union[int, str, list[int]]) -> Any:
+ if isinstance(key, str):
+ return Column(self, key)
+ elif isinstance(self.source, Dataset):
+ if self.source._format_type == "custom":
+ # the formatting transform may require all columns
+ source = self.source
+ else:
+ source = self.source._fast_select_column(self.column_name)
+ return source[key][self.column_name]
+ elif isinstance(key, int):
+ return self.source[key][self.column_name]
+ else:
+ return [item[self.column_name] for item in self.source[key]]
+
+ def __len__(self) -> int:
+ return len(self.source)
+
+ def __repr__(self):
+ return "Column(" + repr(list(self[:5]))[:-1] + (", ...])" if len(self) > 5 else "])")
+
+ def __str__(self):
+ return "Column(" + str(list(self[:5]))[:-1] + (", ...])" if len(self) > 5 else "])")
+
+ def __eq__(self, value):
+ if isinstance(value, Column):
+ return list(self) == list(value)
+ else:
+ return value == list(self)
+
+
+class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin):
+ """A Dataset backed by an Arrow table."""
+
+ def __init__(
+ self,
+ arrow_table: Table,
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ indices_table: Optional[Table] = None,
+ fingerprint: Optional[str] = None,
+ ):
+ info = info.copy() if info is not None else DatasetInfo()
+ DatasetInfoMixin.__init__(self, info=info, split=split)
+ IndexableMixin.__init__(self)
+
+ self._data: Table = _check_table(arrow_table)
+ self._indices: Optional[Table] = _check_table(indices_table) if indices_table is not None else None
+ maybe_register_dataset_for_temp_dir_deletion(self)
+
+ self._format_type: Optional[str] = None
+ self._format_kwargs: dict = {}
+ self._format_columns: Optional[list] = None
+ self._output_all_columns: bool = False
+ self._fingerprint: str = fingerprint
+
+ # Read metadata
+
+ if self._data.schema.metadata is not None and b"huggingface" in self._data.schema.metadata:
+ metadata = json.loads(self._data.schema.metadata[b"huggingface"].decode())
+ if (
+ "fingerprint" in metadata and self._fingerprint is None
+ ): # try to load fingerprint from the arrow file metadata
+ self._fingerprint = metadata["fingerprint"]
+
+ # Infer features if None
+ inferred_features = Features.from_arrow_schema(arrow_table.schema)
+ if self.info.features is None:
+ self.info.features = inferred_features
+ else: # make sure the nested columns are in the right order
+ try:
+ self.info.features = self.info.features.reorder_fields_as(inferred_features)
+ except ValueError as e:
+ raise ValueError(
+ f"{e}\nThe 'source' features come from dataset_info.json, and the 'target' ones are those of the dataset arrow file."
+ )
+
+ # In case there are types like pa.dictionary that we need to convert to the underlying type
+
+ if self.data.schema != self.info.features.arrow_schema:
+ self._data = self.data.cast(self.info.features.arrow_schema)
+
+ # Infer fingerprint if None
+
+ if self._fingerprint is None:
+ self._fingerprint = generate_fingerprint(self)
+
+ # Sanity checks
+
+ if self._info.features is None:
+ raise ValueError("Features can't be None in a Dataset object")
+ if self._fingerprint is None:
+ raise ValueError("Fingerprint can't be None in a Dataset object")
+ if self.info.features.type != inferred_features.type:
+ raise ValueError(
+ f"External features info don't match the dataset:\nGot\n{self.info.features}\nwith type\n{self.info.features.type}\n\nbut expected something like\n{inferred_features}\nwith type\n{inferred_features.type}"
+ )
+
+ if self._indices is not None:
+ if not pa.types.is_unsigned_integer(self._indices.column(0).type):
+ raise ValueError(
+ f"indices must be an Arrow table of unsigned integers, current type is {self._indices.column(0).type}"
+ )
+ _check_column_names(self._data.column_names)
+
+ self._data = update_metadata_with_features(self._data, self._info.features)
+
+ @property
+ def features(self) -> Features:
+ features = super().features
+ if features is None: # this is already checked in __init__
+ raise ValueError("Features can't be None in a Dataset object")
+ return features
+
+ @classmethod
+ def from_file(
+ cls,
+ filename: str,
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ indices_filename: Optional[str] = None,
+ in_memory: bool = False,
+ ) -> "Dataset":
+ """Instantiate a Dataset backed by an Arrow table at filename.
+
+ Args:
+ filename (`str`):
+ File name of the dataset.
+ info (`DatasetInfo`, *optional*):
+ Dataset information, like description, citation, etc.
+ split (`NamedSplit`, *optional*):
+ Name of the dataset split.
+ indices_filename (`str`, *optional*):
+ File names of the indices.
+ in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+
+ Returns:
+ [`Dataset`]
+ """
+ table = ArrowReader.read_table(filename, in_memory=in_memory)
+
+ if indices_filename is not None:
+ indices_pa_table = ArrowReader.read_table(indices_filename, in_memory=in_memory)
+ else:
+ indices_pa_table = None
+
+ return cls(
+ arrow_table=table,
+ info=info,
+ split=split,
+ indices_table=indices_pa_table,
+ )
+
+ @classmethod
+ def from_buffer(
+ cls,
+ buffer: pa.Buffer,
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ indices_buffer: Optional[pa.Buffer] = None,
+ ) -> "Dataset":
+ """Instantiate a Dataset backed by an Arrow buffer.
+
+ Args:
+ buffer (`pyarrow.Buffer`):
+ Arrow buffer.
+ info (`DatasetInfo`, *optional*):
+ Dataset information, like description, citation, etc.
+ split (`NamedSplit`, *optional*):
+ Name of the dataset split.
+ indices_buffer (`pyarrow.Buffer`, *optional*):
+ Indices Arrow buffer.
+
+ Returns:
+ [`Dataset`]
+ """
+ table = InMemoryTable.from_buffer(buffer)
+
+ if indices_buffer is not None:
+ indices_table = InMemoryTable.from_buffer(buffer)
+ else:
+ indices_table = None
+
+ return cls(table, info=info, split=split, indices_table=indices_table)
+
+ @classmethod
+ def from_pandas(
+ cls,
+ df: pd.DataFrame,
+ features: Optional[Features] = None,
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ preserve_index: Optional[bool] = None,
+ ) -> "Dataset":
+ """
+ Convert `pandas.DataFrame` to a `pyarrow.Table` to create a [`Dataset`].
+
+ The column types in the resulting Arrow Table are inferred from the dtypes of the `pandas.Series` in the
+ DataFrame. In the case of non-object Series, the NumPy dtype is translated to its Arrow equivalent. In the
+ case of `object`, we need to guess the datatype by looking at the Python objects in this Series.
+
+ Be aware that Series of the `object` dtype don't carry enough information to always lead to a meaningful Arrow
+ type. In the case that we cannot infer a type, e.g. because the DataFrame is of length 0 or the Series only
+ contains `None/nan` objects, the type is set to `null`. This behavior can be avoided by constructing explicit
+ features and passing it to this function.
+
+ Important: a dataset created with from_pandas() lives in memory
+ and therefore doesn't have an associated cache directory.
+ This may change in the future, but in the meantime if you
+ want to reduce memory usage you should write it back on disk
+ and reload using e.g. save_to_disk / load_from_disk.
+
+ Args:
+ df (`pandas.DataFrame`):
+ Dataframe that contains the dataset.
+ features ([`Features`], *optional*):
+ Dataset features.
+ info (`DatasetInfo`, *optional*):
+ Dataset information, like description, citation, etc.
+ split (`NamedSplit`, *optional*):
+ Name of the dataset split.
+ preserve_index (`bool`, *optional*):
+ Whether to store the index as an additional column in the resulting Dataset.
+ The default of `None` will store the index as a column, except for `RangeIndex` which is stored as metadata only.
+ Use `preserve_index=True` to force it to be stored as a column.
+
+ Returns:
+ [`Dataset`]
+
+ Example:
+
+ ```py
+ >>> ds = Dataset.from_pandas(df)
+ ```
+ """
+ if info is not None and features is not None and info.features != features:
+ raise ValueError(
+ f"Features specified in `features` and `info.features` can't be different:\n{features}\n{info.features}"
+ )
+ features = features if features is not None else info.features if info is not None else None
+ if features is not None:
+ features = _fix_for_backward_compatible_features(features)
+ if info is None:
+ info = DatasetInfo()
+ info.features = features
+ table = InMemoryTable.from_pandas(
+ df=df,
+ preserve_index=preserve_index,
+ )
+ if features is not None:
+ # more expensive cast than InMemoryTable.from_pandas(..., schema=features.arrow_schema)
+ # needed to support the str to Audio conversion for instance
+ table = table.cast(features.arrow_schema)
+ return cls(table, info=info, split=split)
+
+ @classmethod
+ def from_polars(
+ cls,
+ df: Union["pl.DataFrame", "pl.LazyFrame"],
+ features: Optional[Features] = None,
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ ) -> "Dataset":
+ """
+ Collect the underlying arrow arrays in an Arrow Table.
+
+ This operation is mostly zero copy.
+
+ Data types that do copy:
+ * CategoricalType
+
+ Args:
+ df (`polars.DataFrame`): DataFrame to convert to Arrow Table
+ features (`Features`, optional): Dataset features.
+ info (`DatasetInfo`, optional): Dataset information, like description, citation, etc.
+ split (`NamedSplit`, optional): Name of the dataset split.
+
+ Examples:
+ ```py
+ >>> ds = Dataset.from_polars(df)
+ ```
+ """
+ import polars as pl
+
+ if info is not None and features is not None and info.features != features:
+ raise ValueError(
+ f"Features specified in `features` and `info.features` can't be different:\n{features}\n{info.features}"
+ )
+ features = features if features is not None else info.features if info is not None else None
+ if features is not None:
+ features = _fix_for_backward_compatible_features(features)
+ if info is None:
+ info = DatasetInfo()
+ info.features = features
+ if isinstance(df, pl.LazyFrame):
+ df = df.collect()
+ table = InMemoryTable(df.to_arrow())
+ if features is not None:
+ # more expensive cast than InMemoryTable.from_polars(..., schema=features.arrow_schema)
+ # needed to support the str to Audio conversion for instance
+ table = table.cast(features.arrow_schema)
+ return cls(table, info=info, split=split)
+
+ @classmethod
+ def from_dict(
+ cls,
+ mapping: dict,
+ features: Optional[Features] = None,
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ on_mixed_types: Optional[Literal["use_json"]] = None,
+ ) -> "Dataset":
+ """
+ Convert `dict` to a `pyarrow.Table` to create a [`Dataset`].
+
+ Important: a dataset created with from_dict() lives in memory
+ and therefore doesn't have an associated cache directory.
+ This may change in the future, but in the meantime if you
+ want to reduce memory usage you should write it back on disk
+ and reload using e.g. save_to_disk / load_from_disk.
+
+ Args:
+ mapping (`Mapping`):
+ Mapping of strings to Arrays or Python lists.
+ features ([`Features`], *optional*):
+ Dataset features.
+ info (`DatasetInfo`, *optional*):
+ Dataset information, like description, citation, etc.
+ split (`NamedSplit`, *optional*):
+ Name of the dataset split.
+ on_mixed_types (`Literal["use_json"]`, *optional*, defaults to `None`):
+ If "use_json", use the Json() type for mixed-types fields,
+ i.e. unstructured fields that contain data without a predefined schema.
+ In this case, a field with mixed type is set to Json().
+
+ This allow loading lists with a mix of strings/integers/floats
+ for example, or dictionaries with arbitrary value types.
+
+
+
+ Returns:
+ [`Dataset`]
+
+ Examples:
+
+ Get a Dataset from a dictionary containing one list per column:
+
+ ```py
+ >>> ds = Dataset.from_dict({"text": ["hello there !", "general kenobi !"]})
+ ```
+
+ Pass features to set the column types, e.g. for an image dataset:
+
+ ```py
+ >>> features = Features({"image": Image()})
+ >>> ds = Dataset.from_dict({"image": ["path/to/image.png"]}, features=features)
+ ```
+
+ Datasets are based on Arrow which is a columnar format, and therefore they expect every example to have the same
+ type and subtypes, and dictionaries to have the same keys and values types.
+ Loading a dataset errors out when fields have mismatching types, and fills missing fields in dictionaries with None so all dictionaries have the same keys and value types.
+
+ To avoid this and allow mixed-types without errors, you can use `on_mixed_types="use_json"` or specify `features=` with a [`Json`] type:
+
+ ```py
+ >>> ds = Dataset.from_dict({"a": [0, "foo", {"subfield": "bar"}]})
+ Traceback (most recent call last):
+ ...
+ File "pyarrow/error.pxi", line 92, in pyarrow.lib.check_status
+ pyarrow.lib.ArrowInvalid: Could not convert 'foo' with type str: tried to convert to int64
+
+ >>> ds = Dataset.from_dict({"a": [0, "foo", {"subfield": "bar"}]}, on_mixed_types="use_json")
+ >>> ds.features
+ {'a': Json()}
+ >>> list(ds["a"])
+ [0, "foo", {"subfield": "bar"}]
+
+ >>> features = Features({"a": Json()})
+ >>> ds = Dataset.from_dict({"a": [0, "foo", {"subfield": "bar"}]}, features=features)
+ >>> ds.features
+ {'a': Json()}
+ >>> list(ds["a"])
+ [0, "foo", {"subfield": "bar"}]
+ ```
+
+ This is also useful for lists of dictionaries with arbitrary keys and values, to avoid filling missing fields with None:
+
+ ```py
+ >>> ds = Dataset.from_dict({"a": [[{"b": 0}, {"c": 0}]]})
+ >>> ds.features
+ {'a': List({'b': Value('int64'), 'c': Value('int64')})}
+ >>> list(ds["a"])
+ [[{'b': 0, 'c': None}, {'b': None, 'c': 0}]] # missing fields are filled with None
+
+ >>> features = Features({"a": List(Json())})
+ >>> ds = Dataset.from_dict({"a": [[{"b": 0}, {"c": 0}]]}, features=features)
+ >>> ds.features
+ {'a': List(Json())}
+ >>> list(ds["a"])
+ [[{'b': 0}, {'c': 0}]] # OK
+
+ >>> ds = Dataset.from_dict({"a": [[{"b": 0}, {"c": 0}]]}, on_mixed_types="use_json")
+ >>> ds.features
+ {'a': List(Json())}
+ >>> list(ds["a"])
+ [[{'b': 0}, {'c': 0}]] # OK
+ ```
+
+ Another example with tool calling data:
+
+ ```py
+ >>> messages = [
+ ... {"role": "user", "content": "Turn on the living room lights and play my electronic music playlist."},
+ ... {"role": "assistant", "tool_calls": [
+ ... {"type": "function", "function": {
+ ... "name": "control_light",
+ ... "arguments": {"room": "living room", "state": "on"}
+ ... }},
+ ... {"type": "function", "function": {
+ ... "name": "play_music",
+ ... "arguments": {"playlist": "electronic"} # mixed-type here since keys ["playlist"] and ["room", "state"] are different
+ ... }}]
+ ... },
+ ... {"role": "tool", "name": "control_light", "content": "The lights in the living room are now on."},
+ ... {"role": "tool", "name": "play_music", "content": "The music is now playing."},
+ ... {"role": "assistant", "content": "Done!"}
+ ... ]
+ >>> ds = Dataset.from_dict({"messages": [messages]}, on_mixed_types="use_json")
+ >>> ds.features
+ {'messages': List({'role': Value('string'), 'content': Value('string'), 'tool_calls': List(Json()), 'name': Value('string')})}
+ >>> ds[0]["messages"][1]["tool_calls"][0]["function"]["arguments"]
+ {"room": "living room", "state": "on"}
+ ```
+ """
+ if info is not None and features is not None and info.features != features:
+ raise ValueError(
+ f"Features specified in `features` and `info.features` can't be different:\n{features}\n{info.features}"
+ )
+ features = features if features is not None else info.features if info is not None else None
+ if features is not None:
+ features = _fix_for_backward_compatible_features(features)
+ arrow_typed_mapping = {}
+ for col, data in mapping.items():
+ if isinstance(data, (pa.Array, pa.ChunkedArray)):
+ data = cast_array_to_feature(data, features[col]) if features is not None else data
+ else:
+ data = OptimizedTypedSequence(
+ features.encode_column(data, col) if features is not None else data,
+ type=features[col] if features is not None else None,
+ col=col,
+ on_mixed_types=on_mixed_types,
+ )
+ arrow_typed_mapping[col] = data
+ mapping = arrow_typed_mapping
+ pa_table = InMemoryTable.from_pydict(mapping=mapping)
+ if info is None:
+ info = DatasetInfo()
+ info.features = features
+ if info.features is None:
+ info.features = Features(
+ {
+ col: generate_from_arrow_type(data.type)
+ if isinstance(data, (pa.Array, pa.ChunkedArray))
+ else data.get_inferred_type()
+ for col, data in mapping.items()
+ }
+ )
+ return cls(pa_table, info=info, split=split)
+
+ @classmethod
+ def from_list(
+ cls,
+ mapping: list[dict],
+ features: Optional[Features] = None,
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ on_mixed_types: Optional[Literal["use_json"]] = None,
+ ) -> "Dataset":
+ """
+ Convert a list of dicts to a `pyarrow.Table` to create a [`Dataset`]`.
+
+ Note that the keys of the first entry will be used to determine the dataset columns,
+ regardless of what is passed to features.
+
+ Important: a dataset created with from_list() lives in memory
+ and therefore doesn't have an associated cache directory.
+ This may change in the future, but in the meantime if you
+ want to reduce memory usage you should write it back on disk
+ and reload using e.g. save_to_disk / load_from_disk.
+
+ Args:
+ mapping (`List[dict]`): A list of mappings of strings to row values.
+ features (`Features`, optional): Dataset features.
+ info (`DatasetInfo`, optional): Dataset information, like description, citation, etc.
+ split (`NamedSplit`, optional): Name of the dataset split.
+ on_mixed_types (`Literal["use_json"]`, *optional*, defaults to `None`):
+ If "use_json", use the Json() type for mixed-types fields,
+ i.e. unstructured fields that contain data without a predefined schema.
+ In this case, a field with mixed type is set to Json().
+
+ This allow loading lists with a mix of strings/integers/floats
+ for example, or dictionaries with arbitrary value types.
+
+
+
+ Returns:
+ [`Dataset`]
+
+ Examples:
+
+ Get a Dataset from a list containing the examples:
+
+ ```py
+ >>> ds = Dataset.from_list([{"text": "hello there !"}, {"text": "general kenobi !"}]})
+ ```
+
+ Pass features to set the column types, e.g. for an image dataset:
+
+ ```py
+ >>> features = Features({"image": Image()})
+ >>> ds = Dataset.from_list([{"image": "path/to/image.png"}], features=features)
+ ```
+
+ Datasets are based on Arrow which is a columnar format, and therefore they expect every example to have the same
+ type and subtypes, and dictionaries to have the same keys and values types.
+ Loading a dataset errors out when fields have mismatching types, and fills missing fields in dictionaries with None so all dictionaries have the same keys and value types.
+
+ To avoid this and allow mixed-types without errors, you can use `on_mixed_types="use_json"` or specify `features=` with a [`Json`] type:
+
+ ```py
+ >>> ds = Dataset.from_list([{"a": 0}, {"a": "foo"}, {"a": {"subfield": "bar"}}])
+ Traceback (most recent call last):
+ ...
+ File "pyarrow/error.pxi", line 92, in pyarrow.lib.check_status
+ pyarrow.lib.ArrowInvalid: Could not convert 'foo' with type str: tried to convert to int64
+
+ >>> ds = Dataset.from_list([{"a": 0}, {"a": "foo"}, {"a": {"subfield": "bar"}}], on_mixed_types="use_json")
+ >>> ds.features
+ {'a': Json()}
+ >>> list(ds["a"])
+ [0, "foo", {"subfield": "bar"}]
+
+ >>> features = Features({"a": Json()})
+ >>> ds = Dataset.from_list([{"a": 0}, {"a": "foo"}, {"a": {"subfield": "bar"}}], features=features)
+ >>> ds.features
+ {'a': Json()}
+ >>> list(ds["a"])
+ [0, "foo", {"subfield": "bar"}]
+ ```
+
+ This is also useful for lists of dictionaries with arbitrary keys and values, to avoid filling missing fields with None:
+
+ ```py
+ >>> ds = Dataset.from_list([{"a": [{"b": 0}, {"c": 0}]}])
+ >>> ds.features
+ {'a': List({'b': Value('int64'), 'c': Value('int64')})}
+ >>> list(ds["a"])
+ [[{'b': 0, 'c': None}, {'b': None, 'c': 0}]] # missing fields are filled with None
+
+ >>> features = Features({"a": List(Json())})
+ >>> ds = Dataset.from_list([{"a": [{"b": 0}, {"c": 0}]}], features=features)
+ >>> ds.features
+ {'a': List(Json())}
+ >>> list(ds["a"])
+ [[{'b': 0}, {'c': 0}]] # OK
+
+ >>> ds = Dataset.from_list([{"a": [{"b": 0}, {"c": 0}]}], on_mixed_types="use_json")
+ >>> ds.features
+ {'a': List(Json())}
+ >>> list(ds["a"])
+ [[{'b': 0}, {'c': 0}]] # OK
+ ```
+
+ Another example with tool calling data:
+
+ ```py
+ >>> messages = [
+ ... {"role": "user", "content": "Turn on the living room lights and play my electronic music playlist."},
+ ... {"role": "assistant", "tool_calls": [
+ ... {"type": "function", "function": {
+ ... "name": "control_light",
+ ... "arguments": {"room": "living room", "state": "on"}
+ ... }},
+ ... {"type": "function", "function": {
+ ... "name": "play_music",
+ ... "arguments": {"playlist": "electronic"} # mixed-type here since keys ["playlist"] and ["room", "state"] are different
+ ... }}]
+ ... },
+ ... {"role": "tool", "name": "control_light", "content": "The lights in the living room are now on."},
+ ... {"role": "tool", "name": "play_music", "content": "The music is now playing."},
+ ... {"role": "assistant", "content": "Done!"}
+ ... ]
+ >>> ds = Dataset.from_list([{"messages": messages}], on_mixed_types="use_json")
+ >>> ds.features
+ {'messages': List({'role': Value('string'), 'content': Value('string'), 'tool_calls': List(Json()), 'name': Value('string')})}
+ >>> ds[0]["messages"][1]["tool_calls"][0]["function"]["arguments"]
+ {"room": "living room", "state": "on"}
+ ```
+ """
+ # for simplicity and consistency wrt OptimizedTypedSequence we do not use InMemoryTable.from_pylist here
+ mapping = {k: [r.get(k) for r in mapping] for k in mapping[0]} if mapping else {}
+ return cls.from_dict(mapping, features, info, split, on_mixed_types=on_mixed_types)
+
+ @staticmethod
+ def from_csv(
+ path_or_paths: Union[PathLike, list[PathLike]],
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ num_proc: Optional[int] = None,
+ **kwargs,
+ ) -> "Dataset":
+ """Create Dataset from CSV file(s).
+
+ Read the CSV files, cache the data in Arrow format on disk and return the Dataset from the memory-mapped Arrow data on disk.
+
+ Args:
+ path_or_paths (`path-like` or list of `path-like`):
+ Path(s) of the CSV file(s).
+ split ([`NamedSplit`], *optional*):
+ Split name to be assigned to the dataset.
+ features ([`Features`], *optional*):
+ Dataset features.
+ cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`):
+ Directory to cache data.
+ keep_in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+ num_proc (`int`, *optional*, defaults to `None`):
+ Number of processes when downloading and generating the dataset locally.
+ This is helpful if the dataset is made of multiple files. Multiprocessing is disabled by default.
+
+
+ **kwargs (additional keyword arguments):
+ Keyword arguments to be passed to [`pandas.read_csv`].
+
+ Returns:
+ [`Dataset`]
+
+ Example:
+
+ ```py
+ >>> ds = Dataset.from_csv('path/to/dataset.csv')
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.csv import CsvDatasetReader
+
+ return CsvDatasetReader(
+ path_or_paths,
+ split=split,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ num_proc=num_proc,
+ **kwargs,
+ ).read()
+
+ @staticmethod
+ def from_generator(
+ generator: Callable,
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ gen_kwargs: Optional[dict] = None,
+ num_proc: Optional[int] = None,
+ split: NamedSplit = Split.TRAIN,
+ fingerprint: Optional[str] = None,
+ **kwargs,
+ ) -> "Dataset":
+ """Create a Dataset from a generator.
+
+ Load the data from the generator, cache the data in Arrow format on disk and return the Dataset from the memory-mapped Arrow data on disk.
+
+ Args:
+ generator (:`Callable`):
+ A generator function that `yields` examples.
+ features ([`Features`], *optional*):
+ Dataset features.
+ cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`):
+ Directory to cache data.
+ keep_in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+ gen_kwargs(`dict`, *optional*):
+ Keyword arguments to be passed to the `generator` callable.
+ You can define a sharded dataset by passing the list of shards in `gen_kwargs` and setting `num_proc` greater than 1.
+ num_proc (`int`, *optional*, defaults to `None`):
+ Number of processes when downloading and generating the dataset locally.
+ This is helpful if the dataset is made of multiple files. Multiprocessing is disabled by default.
+ If `num_proc` is greater than one, then all list values in `gen_kwargs` must be the same length. These values will be split between calls to the generator. The number of shards will be the minimum of the shortest list in `gen_kwargs` and `num_proc`.
+
+
+ split ([`NamedSplit`], defaults to `Split.TRAIN`):
+ Split name to be assigned to the dataset.
+
+
+ fingerprint (`str`, *optional*):
+ Fingerprint that will be used to generate dataset ID.
+ By default `fingerprint` is generated by hashing the generator function and all the args which can be slow
+ if it uses large objects like AI models.
+
+
+ **kwargs (additional keyword arguments):
+ Keyword arguments to be passed to :[`GeneratorConfig`].
+
+ Returns:
+ [`Dataset`]
+
+ Example:
+
+ ```py
+ >>> def gen():
+ ... yield {"text": "Good", "label": 0}
+ ... yield {"text": "Bad", "label": 1}
+ ...
+ >>> ds = Dataset.from_generator(gen)
+ ```
+
+ ```py
+ >>> def gen(shards):
+ ... for shard in shards:
+ ... with open(shard) as f:
+ ... for line in f:
+ ... yield {"line": line}
+ ...
+ >>> shards = [f"data{i}.txt" for i in range(32)]
+ >>> ds = Dataset.from_generator(gen, gen_kwargs={"shards": shards})
+ ```
+ """
+ from .io.generator import GeneratorDatasetInputStream
+
+ return GeneratorDatasetInputStream(
+ generator=generator,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ gen_kwargs=gen_kwargs,
+ num_proc=num_proc,
+ split=split,
+ fingerprint=fingerprint,
+ **kwargs,
+ ).read()
+
+ @staticmethod
+ def from_json(
+ path_or_paths: Union[PathLike, list[PathLike]],
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ field: Optional[str] = None,
+ num_proc: Optional[int] = None,
+ **kwargs,
+ ) -> "Dataset":
+ """Create Dataset from JSON or JSON Lines file(s).
+
+ Read the JSON files, cache the data in Arrow format on disk and return the Dataset from the memory-mapped Arrow data on disk.
+
+ Args:
+ path_or_paths (`path-like` or list of `path-like`):
+ Path(s) of the JSON or JSON Lines file(s).
+ split ([`NamedSplit`], *optional*):
+ Split name to be assigned to the dataset.
+ features ([`Features`], *optional*):
+ Dataset features.
+ cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`):
+ Directory to cache data.
+ keep_in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+ field (`str`, *optional*):
+ Field name of the JSON file where the dataset is contained in.
+ num_proc (`int`, *optional* defaults to `None`):
+ Number of processes when downloading and generating the dataset locally.
+ This is helpful if the dataset is made of multiple files. Multiprocessing is disabled by default.
+
+
+ **kwargs (additional keyword arguments):
+ Keyword arguments to be passed to [`JsonConfig`].
+
+ Returns:
+ [`Dataset`]
+
+ Example:
+
+ ```py
+ >>> ds = Dataset.from_json('path/to/dataset.json')
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.json import JsonDatasetReader
+
+ return JsonDatasetReader(
+ path_or_paths,
+ split=split,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ field=field,
+ num_proc=num_proc,
+ **kwargs,
+ ).read()
+
+ @staticmethod
+ def from_parquet(
+ path_or_paths: Union[PathLike, list[PathLike]],
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ columns: Optional[list[str]] = None,
+ num_proc: Optional[int] = None,
+ filters: Optional[Union[pds.Expression, list[tuple], list[list[tuple]]]] = None,
+ fragment_scan_options: Optional[pds.ParquetFragmentScanOptions] = None,
+ on_bad_files: Literal["error", "warn", "skip"] = "error",
+ **kwargs,
+ ) -> "Dataset":
+ """Create Dataset from Parquet file(s).
+
+ Read the Parquet files, cache the data in Arrow format on disk and return the Dataset from the memory-mapped Arrow data on disk.
+
+ Args:
+ path_or_paths (`path-like` or list of `path-like`):
+ Path(s) of the Parquet file(s).
+ split (`NamedSplit`, *optional*):
+ Split name to be assigned to the dataset.
+ features (`Features`, *optional*):
+ Dataset features.
+ cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`):
+ Directory to cache data.
+ keep_in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+ columns (`List[str]`, *optional*):
+ If not `None`, only these columns will be read from the file.
+ A column name may be a prefix of a nested field, e.g. 'a' will select
+ 'a.b', 'a.c', and 'a.d.e'.
+ num_proc (`int`, *optional*, defaults to `None`):
+ Number of processes when downloading and generating the dataset locally.
+ This is helpful if the dataset is made of multiple files. Multiprocessing is disabled by default.
+
+
+ filters (`Union[pyarrow.dataset.Expression, list[tuple], list[list[tuple]]]`, *optional*):
+ Return only the rows matching the filter.
+ If possible the predicate will be pushed down to exploit the partition information
+ or internal metadata found in the data source, e.g. Parquet statistics.
+ Otherwise filters the loaded RecordBatches before yielding them.
+ fragment_scan_options (`pyarrow.dataset.ParquetFragmentScanOptions`, *optional*)
+ Scan-specific options for Parquet fragments.
+ This is especially useful to configure buffering and caching.
+
+
+ on_bad_files (`Literal["error", "warn", "skip"]`, *optional*, defaults to "error")
+ Specify what to do upon encountering a bad file (a file that can't be read). Allowed values are :
+ * 'error', raise an Exception when a bad file is encountered.
+ * 'warn', raise a warning when a bad file is encountered and skip that file.
+ * 'skip', skip bad files without raising or warning when they are encountered.
+
+
+ **kwargs (additional keyword arguments):
+ Keyword arguments to be passed to [`ParquetConfig`].
+
+ Returns:
+ [`Dataset`]
+
+ Example:
+
+ ```py
+ >>> ds = Dataset.from_parquet('path/to/dataset.parquet')
+ ```
+
+ Load a subset of columns:
+
+ ```python
+ >>> ds = Dataset.from_parquet('path/to/dataset.parquet', columns=["col_0", "col_1"])
+ ```
+
+ Efficiently filter data, possibly skipping entire files or row groups:
+
+ ```python
+ >>> filters = [("col_0", "==", 0)]
+ >>> ds = Dataset.from_parquet(parquet_files_list, filters=filters)
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.parquet import ParquetDatasetReader
+
+ return ParquetDatasetReader(
+ path_or_paths,
+ split=split,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ columns=columns,
+ num_proc=num_proc,
+ filters=filters,
+ fragment_scan_options=fragment_scan_options,
+ on_bad_files=on_bad_files,
+ **kwargs,
+ ).read()
+
+ @staticmethod
+ def from_text(
+ path_or_paths: Union[PathLike, list[PathLike]],
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ num_proc: Optional[int] = None,
+ keep_linebreaks: bool = False,
+ sample_by: Literal["line", "paragraph", "document"] = "line",
+ **kwargs,
+ ) -> "Dataset":
+ """Create Dataset from text file(s).
+
+ Read the text files, cache the data in Arrow format on disk and return the Dataset from the memory-mapped Arrow data on disk.
+
+ Args:
+ path_or_paths (`path-like` or list of `path-like`):
+ Path(s) of the text file(s).
+ split (`NamedSplit`, *optional*):
+ Split name to be assigned to the dataset.
+ features (`Features`, *optional*):
+ Dataset features.
+ cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`):
+ Directory to cache data.
+ keep_in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+ num_proc (`int`, *optional*, defaults to `None`):
+ Number of processes when downloading and generating the dataset locally.
+ This is helpful if the dataset is made of multiple files. Multiprocessing is disabled by default.
+
+
+ keep_linebreaks: (`bool`, defaults to False):
+ Whether to keep line breaks.
+ sample_by (`Literal["line", "paragraph", "document"]`, defaults to "line"):
+ Whether to load data per line, praragraph or document.
+ By default one row in the dataset = one line.
+ **kwargs (additional keyword arguments):
+ Keyword arguments to be passed to [`TextConfig`].
+
+ Returns:
+ [`Dataset`]
+
+ Example:
+
+ ```py
+ >>> ds = Dataset.from_text('path/to/dataset.txt')
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.text import TextDatasetReader
+
+ return TextDatasetReader(
+ path_or_paths,
+ split=split,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ num_proc=num_proc,
+ keep_linebreaks=keep_linebreaks,
+ sample_by=sample_by,
+ **kwargs,
+ ).read()
+
+ @staticmethod
+ def from_spark(
+ df: "pyspark.sql.DataFrame",
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ keep_in_memory: bool = False,
+ cache_dir: str = None,
+ working_dir: str = None,
+ load_from_cache_file: bool = True,
+ **kwargs,
+ ) -> "Dataset":
+ """Create a Dataset from Spark DataFrame. Dataset downloading is distributed over Spark workers.
+
+ Read the Spark DataFrame, cache the data in Arrow format on disk and return the Dataset from the memory-mapped Arrow data on disk.
+
+ Args:
+ df (`pyspark.sql.DataFrame`):
+ The DataFrame containing the desired data.
+ split (`NamedSplit`, *optional*):
+ Split name to be assigned to the dataset.
+ features (`Features`, *optional*):
+ Dataset features.
+ cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`):
+ Directory to cache data. When using a multi-node Spark cluster, the cache_dir must be accessible to both
+ workers and the driver.
+ keep_in_memory (`bool`):
+ Whether to copy the data in-memory.
+ working_dir (`str`, *optional*)
+ Intermediate directory for each Spark worker to write data to before moving it to `cache_dir`. Setting
+ a non-NFS intermediate directory may improve performance.
+ load_from_cache_file (`bool`):
+ Whether to load the dataset from the cache if possible.
+
+ Returns:
+ [`Dataset`]
+
+ Example:
+
+ ```py
+ >>> df = spark.createDataFrame(
+ >>> data=[[1, "Elia"], [2, "Teo"], [3, "Fang"]],
+ >>> columns=["id", "name"],
+ >>> )
+ >>> ds = Dataset.from_spark(df)
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.spark import SparkDatasetReader
+
+ if sys.platform == "win32":
+ raise OSError("Dataset.from_spark is not currently supported on Windows")
+
+ return SparkDatasetReader(
+ df,
+ split=split,
+ features=features,
+ streaming=False,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ working_dir=working_dir,
+ load_from_cache_file=load_from_cache_file,
+ **kwargs,
+ ).read()
+
+ @staticmethod
+ def from_sql(
+ sql: Union[str, "sqlalchemy.sql.Selectable"],
+ con: Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"],
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ **kwargs,
+ ) -> "Dataset":
+ """Create Dataset from SQL query or database table.
+
+ Query the SQL database, cache the data in Arrow format on disk and return the Dataset from the memory-mapped Arrow data on disk.
+
+ Args:
+ sql (`str` or `sqlalchemy.sql.Selectable`):
+ SQL query to be executed or a table name.
+ con (`str` or `sqlite3.Connection` or `sqlalchemy.engine.Connection` or `sqlalchemy.engine.Connection`):
+ A [URI string](https://docs.sqlalchemy.org/en/13/core/engines.html#database-urls) used to instantiate a database connection or a SQLite3/SQLAlchemy connection object.
+ features ([`Features`], *optional*):
+ Dataset features.
+ cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`):
+ Directory to cache data.
+ keep_in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+ **kwargs (additional keyword arguments):
+ Keyword arguments to be passed to [`SqlConfig`].
+
+ Returns:
+ [`Dataset`]
+
+ Example:
+
+ ```py
+ >>> # Fetch a database table
+ >>> ds = Dataset.from_sql("test_data", "postgres:///db_name")
+ >>> # Execute a SQL query on the table
+ >>> ds = Dataset.from_sql("SELECT sentence FROM test_data", "postgres:///db_name")
+ >>> # Use a Selectable object to specify the query
+ >>> from sqlalchemy import select, text
+ >>> stmt = select([text("sentence")]).select_from(text("test_data"))
+ >>> ds = Dataset.from_sql(stmt, "postgres:///db_name")
+ ```
+
+ > [!TIP]
+ > The returned dataset can only be cached if `con` is specified as URI string.
+ """
+ from .io.sql import SqlDatasetReader
+
+ return SqlDatasetReader(
+ sql,
+ con,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ **kwargs,
+ ).read()
+
+ def __setstate__(self, state):
+ self.__dict__.update(state)
+ maybe_register_dataset_for_temp_dir_deletion(self)
+ return self
+
+ def __del__(self):
+ if hasattr(self, "_data"):
+ del self._data
+ if hasattr(self, "_indices"):
+ del self._indices
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ # Here `del` is used to del the pyarrow tables. This properly closes the files used for memory mapped tables
+ self.__del__()
+
+ def save_to_disk(
+ self,
+ dataset_path: PathLike,
+ max_shard_size: Optional[Union[str, int]] = None,
+ num_shards: Optional[int] = None,
+ num_proc: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ ):
+ """
+ Saves a dataset to a dataset directory, or in a filesystem using any implementation of `fsspec.spec.AbstractFileSystem`.
+
+ For [`Image`], [`Audio`] and [`Video`] data:
+
+ All the Image(), Audio() and Video() data are stored in the arrow files.
+ If you want to store paths or urls, please use the Value("string") type.
+
+ Args:
+ dataset_path (`path-like`):
+ Path (e.g. `dataset/train`) or remote URI (e.g. `s3://my-bucket/dataset/train`)
+ of the dataset directory where the dataset will be saved to.
+ max_shard_size (`int` or `str`, *optional*, defaults to `"500MB"`):
+ The maximum size of the dataset shards to be saved to the filesystem. If expressed as a string, needs to be digits followed by a unit
+ (like `"50MB"`).
+ num_shards (`int`, *optional*):
+ Number of shards to write. By default the number of shards depends on `max_shard_size` and `num_proc`.
+
+
+ num_proc (`int`, *optional*):
+ Number of processes when downloading and generating the dataset locally.
+ Multiprocessing is disabled by default.
+
+
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+
+
+
+ Example:
+
+ ```py
+ >>> ds.save_to_disk("path/to/dataset/directory")
+ >>> ds.save_to_disk("path/to/dataset/directory", max_shard_size="1GB")
+ >>> ds.save_to_disk("path/to/dataset/directory", num_shards=1024)
+ ```
+ """
+ if max_shard_size is not None and num_shards is not None:
+ raise ValueError(
+ "Failed to push_to_hub: please specify either max_shard_size or num_shards, but not both."
+ )
+ if self.list_indexes():
+ raise ValueError("please remove all the indexes using `dataset.drop_index` before saving a dataset")
+
+ if num_shards is None:
+ dataset_nbytes = self._estimate_nbytes()
+ max_shard_size = convert_file_size_to_int(max_shard_size or config.MAX_SHARD_SIZE)
+ num_shards = int(dataset_nbytes / max_shard_size) + 1
+ num_shards = max(num_shards, num_proc or 1)
+ # if we have only a few large samples, we should only create as many shards as samples
+ num_shards = min(len(self.data), num_shards)
+
+ fs: fsspec.AbstractFileSystem
+ fs, _ = url_to_fs(dataset_path, **(storage_options or {}))
+
+ if not is_remote_filesystem(fs):
+ parent_cache_files_paths = {
+ Path(cache_filename["filename"]).resolve().parent for cache_filename in self.cache_files
+ }
+ # Check that the dataset doesn't overwrite itself. It can cause a permission error on Windows and a segfault on linux.
+ if Path(dataset_path).expanduser().resolve() in parent_cache_files_paths:
+ raise PermissionError(
+ f"Tried to overwrite {Path(dataset_path).expanduser().resolve()} but a dataset can't overwrite itself."
+ )
+
+ fs.makedirs(dataset_path, exist_ok=True)
+
+ # Get json serializable state
+ state = {
+ key: self.__dict__[key]
+ for key in [
+ "_fingerprint",
+ "_format_columns",
+ "_format_kwargs",
+ "_format_type",
+ "_output_all_columns",
+ ]
+ }
+ state["_split"] = str(self.split) if self.split is not None else self.split
+ state["_data_files"] = [
+ {"filename": f"data-{shard_idx:05d}-of-{num_shards:05d}.arrow"} for shard_idx in range(num_shards)
+ ]
+ for k in state["_format_kwargs"].keys():
+ try:
+ json.dumps(state["_format_kwargs"][k])
+ except TypeError as e:
+ raise TypeError(
+ str(e) + f"\nThe format kwargs must be JSON serializable, but key '{k}' isn't."
+ ) from None
+ # Get json serializable dataset info
+ dataset_info = asdict(self._info)
+
+ shards_done = 0
+ pbar = hf_tqdm(
+ unit=" examples",
+ total=len(self),
+ desc=f"Saving the dataset ({shards_done}/{num_shards} shards)",
+ )
+ kwargs_per_job = (
+ {
+ "job_id": shard_idx,
+ "shard": self.shard(num_shards=num_shards, index=shard_idx, contiguous=True),
+ "fpath": posixpath.join(dataset_path, f"data-{shard_idx:05d}-of-{num_shards:05d}.arrow"),
+ "storage_options": storage_options,
+ }
+ for shard_idx in range(num_shards)
+ )
+ shard_lengths = [None] * num_shards
+ shard_sizes = [None] * num_shards
+ if num_proc is not None and num_proc >= 1:
+ with mp.get_context("spawn").Pool(num_proc) as pool:
+ with pbar:
+ for job_id, done, content in iflatmap_unordered(
+ pool, Dataset._save_to_disk_single, kwargs_iterable=kwargs_per_job
+ ):
+ if done:
+ shards_done += 1
+ pbar.set_description(f"Saving the dataset ({shards_done}/{num_shards} shards)")
+ logger.debug(f"Finished writing shard number {job_id} of {num_shards}.")
+ shard_lengths[job_id], shard_sizes[job_id] = content
+ else:
+ pbar.update(content)
+ else:
+ with pbar:
+ for kwargs in kwargs_per_job:
+ for job_id, done, content in Dataset._save_to_disk_single(**kwargs):
+ if done:
+ shards_done += 1
+ pbar.set_description(f"Saving the dataset ({shards_done}/{num_shards} shards)")
+ logger.debug(f"Finished writing shard number {job_id} of {num_shards}.")
+ shard_lengths[job_id], shard_sizes[job_id] = content
+ else:
+ pbar.update(content)
+ with fs.open(
+ posixpath.join(dataset_path, config.DATASET_STATE_JSON_FILENAME), "w", encoding="utf-8"
+ ) as state_file:
+ json.dump(state, state_file, indent=2, sort_keys=True)
+ with fs.open(
+ posixpath.join(dataset_path, config.DATASET_INFO_FILENAME), "w", encoding="utf-8"
+ ) as dataset_info_file:
+ # Sort only the first level of keys, or we might shuffle fields of nested features if we use sort_keys=True
+ sorted_keys_dataset_info = {key: dataset_info[key] for key in sorted(dataset_info)}
+ json.dump(sorted_keys_dataset_info, dataset_info_file, indent=2)
+
+ @staticmethod
+ def _save_to_disk_single(job_id: int, shard: "Dataset", fpath: str, storage_options: Optional[dict]):
+ batch_size = config.DEFAULT_MAX_BATCH_SIZE
+
+ num_examples_progress_update = 0
+ writer = ArrowWriter(
+ features=shard.features,
+ path=fpath,
+ storage_options=storage_options,
+ embed_local_files=True,
+ )
+ try:
+ _time = time.time()
+ for pa_table in shard.with_format("arrow").iter(batch_size):
+ writer.write_table(pa_table)
+ num_examples_progress_update += len(pa_table)
+ if time.time() > _time + config.PBAR_REFRESH_TIME_INTERVAL:
+ _time = time.time()
+ yield job_id, False, num_examples_progress_update
+ num_examples_progress_update = 0
+ finally:
+ yield job_id, False, num_examples_progress_update
+ num_examples, num_bytes = writer.finalize()
+ writer.close()
+
+ yield job_id, True, (num_examples, num_bytes)
+
+ @staticmethod
+ def _build_local_temp_path(uri_or_path: str) -> Path:
+ """
+ Builds and returns a Path concatenating a local temporary dir with the dir path (or absolute/relative
+ path extracted from the uri) passed.
+
+ Args:
+ uri_or_path (`str`): Path (e.g. `"dataset/train"`) or remote URI (e.g.
+ `"s3://my-bucket/dataset/train"`) to concatenate.
+
+ Returns:
+ :class:`Path`: the concatenated path (temp dir + path)
+ """
+ src_dataset_path = Path(uri_or_path)
+ tmp_dir = get_temporary_cache_files_directory()
+ return Path(tmp_dir, src_dataset_path.relative_to(src_dataset_path.anchor))
+
+ @staticmethod
+ def load_from_disk(
+ dataset_path: PathLike,
+ keep_in_memory: Optional[bool] = None,
+ storage_options: Optional[dict] = None,
+ ) -> "Dataset":
+ """
+ Loads a dataset that was previously saved using [`save_to_disk`] from a dataset directory, or from a
+ filesystem using any implementation of `fsspec.spec.AbstractFileSystem`.
+
+ Args:
+ dataset_path (`path-like`):
+ Path (e.g. `"dataset/train"`) or remote URI (e.g. `"s3//my-bucket/dataset/train"`)
+ of the dataset directory where the dataset will be loaded from.
+ keep_in_memory (`bool`, defaults to `None`):
+ Whether to copy the dataset in-memory. If `None`, the
+ dataset will not be copied in-memory unless explicitly enabled by setting
+ `datasets.config.IN_MEMORY_MAX_SIZE` to nonzero. See more details in the
+ [improve performance](../cache#improve-performance) section.
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+
+
+
+ Returns:
+ [`Dataset`] or [`DatasetDict`]:
+ - If `dataset_path` is a path of a dataset directory, the dataset requested.
+ - If `dataset_path` is a path of a dataset dict directory, a `datasets.DatasetDict` with each split.
+
+ Example:
+
+ ```py
+ >>> ds = load_from_disk("path/to/dataset/directory")
+ ```
+ """
+ fs: fsspec.AbstractFileSystem
+ fs, dataset_path = url_to_fs(dataset_path, **(storage_options or {}))
+
+ dest_dataset_path = dataset_path
+ dataset_dict_json_path = posixpath.join(dest_dataset_path, config.DATASETDICT_JSON_FILENAME)
+ dataset_state_json_path = posixpath.join(dest_dataset_path, config.DATASET_STATE_JSON_FILENAME)
+ dataset_info_path = posixpath.join(dest_dataset_path, config.DATASET_INFO_FILENAME)
+
+ dataset_dict_is_file = fs.isfile(dataset_dict_json_path)
+ dataset_info_is_file = fs.isfile(dataset_info_path)
+ dataset_state_is_file = fs.isfile(dataset_state_json_path)
+ if not dataset_info_is_file and not dataset_state_is_file:
+ if dataset_dict_is_file:
+ raise FileNotFoundError(
+ f"No such files: '{dataset_info_path}', nor '{dataset_state_json_path}' found. Expected to load a `Dataset` object, but got a `DatasetDict`. Please use either `datasets.load_from_disk` or `DatasetDict.load_from_disk` instead."
+ )
+ raise FileNotFoundError(
+ f"No such files: '{dataset_info_path}', nor '{dataset_state_json_path}' found. Expected to load a `Dataset` object but provided path is not a `Dataset`."
+ )
+ if not dataset_info_is_file:
+ if dataset_dict_is_file:
+ raise FileNotFoundError(
+ f"No such file: '{dataset_info_path}' found. Expected to load a `Dataset` object, but got a `DatasetDict`. Please use either `datasets.load_from_disk` or `DatasetDict.load_from_disk` instead."
+ )
+ raise FileNotFoundError(
+ f"No such file: '{dataset_info_path}'. Expected to load a `Dataset` object but provided path is not a `Dataset`."
+ )
+ if not dataset_state_is_file:
+ if dataset_dict_is_file:
+ raise FileNotFoundError(
+ f"No such file: '{dataset_state_json_path}' found. Expected to load a `Dataset` object, but got a `DatasetDict`. Please use either `datasets.load_from_disk` or `DatasetDict.load_from_disk` instead."
+ )
+ raise FileNotFoundError(
+ f"No such file: '{dataset_state_json_path}'. Expected to load a `Dataset` object but provided path is not a `Dataset`."
+ )
+
+ # copies file from filesystem if it is remote filesystem to local filesystem and modifies dataset_path to temp directory containing local copies
+ if is_remote_filesystem(fs):
+ src_dataset_path = dest_dataset_path
+ dest_dataset_path = Dataset._build_local_temp_path(src_dataset_path)
+ fs.download(src_dataset_path, dest_dataset_path.as_posix(), recursive=True)
+ dataset_state_json_path = posixpath.join(dest_dataset_path, config.DATASET_STATE_JSON_FILENAME)
+ dataset_info_path = posixpath.join(dest_dataset_path, config.DATASET_INFO_FILENAME)
+
+ with open(dataset_state_json_path, encoding="utf-8") as state_file:
+ state = json.load(state_file)
+ with open(dataset_info_path, encoding="utf-8") as dataset_info_file:
+ dataset_info = DatasetInfo.from_dict(json.load(dataset_info_file))
+
+ dataset_size = estimate_dataset_size(
+ Path(dest_dataset_path, data_file["filename"]) for data_file in state["_data_files"]
+ )
+ keep_in_memory = keep_in_memory if keep_in_memory is not None else is_small_dataset(dataset_size)
+ table_cls = InMemoryTable if keep_in_memory else MemoryMappedTable
+
+ arrow_table = concat_tables(
+ thread_map(
+ table_cls.from_file,
+ [posixpath.join(dest_dataset_path, data_file["filename"]) for data_file in state["_data_files"]],
+ tqdm_class=hf_tqdm,
+ desc="Loading dataset from disk",
+ # set `disable=None` rather than `disable=False` by default to disable progress bar when no TTY attached
+ disable=len(state["_data_files"]) <= 16 or None,
+ )
+ )
+
+ split = state["_split"]
+ split = Split(split) if split is not None else split
+
+ dataset = Dataset(
+ arrow_table=arrow_table,
+ info=dataset_info,
+ split=split,
+ fingerprint=state["_fingerprint"],
+ )
+
+ format = {
+ "type": state["_format_type"],
+ "format_kwargs": state["_format_kwargs"],
+ "columns": state["_format_columns"],
+ "output_all_columns": state["_output_all_columns"],
+ }
+ dataset = dataset.with_format(**format)
+
+ return dataset
+
+ @property
+ def data(self) -> Table:
+ """The Apache Arrow table backing the dataset.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds.data
+ MemoryMappedTable
+ text: string
+ label: int64
+ ----
+ text: [["compassionately explores the seemingly irreconcilable situation between conservative christian parents and their estranged gay and lesbian children .","the soundtrack alone is worth the price of admission .","rodriguez does a splendid job of racial profiling hollywood style--casting excellent latin actors of all ages--a trend long overdue .","beneath the film's obvious determination to shock at any cost lies considerable skill and determination , backed by sheer nerve .","bielinsky is a filmmaker of impressive talent .","so beautifully acted and directed , it's clear that washington most certainly has a new career ahead of him if he so chooses .","a visual spectacle full of stunning images and effects .","a gentle and engrossing character study .","it's enough to watch huppert scheming , with her small , intelligent eyes as steady as any noir villain , and to enjoy the perfectly pitched web of tension that chabrol spins .","an engrossing portrait of uncompromising artists trying to create something original against the backdrop of a corporate music industry that only seems to care about the bottom line .",...,"ultimately , jane learns her place as a girl , softens up and loses some of the intensity that made her an interesting character to begin with .","ah-nuld's action hero days might be over .","it's clear why deuces wild , which was shot two years ago , has been gathering dust on mgm's shelf .","feels like nothing quite so much as a middle-aged moviemaker's attempt to surround himself with beautiful , half-naked women .","when the precise nature of matthew's predicament finally comes into sharp focus , the revelation fails to justify the build-up .","this picture is murder by numbers , and as easy to be bored by as your abc's , despite a few whopping shootouts .","hilarious musical comedy though stymied by accents thick as mud .","if you are into splatter movies , then you will probably have a reasonably good time with the salton sea .","a dull , simple-minded and stereotypical tale of drugs , death and mind-numbing indifference on the inner-city streets .","the feature-length stretch . . . strains the show's concept ."]]
+ label: [[1,1,1,1,1,1,1,1,1,1,...,0,0,0,0,0,0,0,0,0,0]]
+ ```
+ """
+ return self._data
+
+ @property
+ def cache_files(self) -> list[dict]:
+ """The cache files containing the Apache Arrow table backing the dataset.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds.cache_files
+ [{'filename': '/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/40d411e45a6ce3484deed7cc15b82a53dad9a72aafd9f86f8f227134bec5ca46/rotten_tomatoes_movie_review-validation.arrow'}]
+ ```
+ """
+ cache_files = list_table_cache_files(self._data)
+ if self._indices is not None:
+ cache_files += list_table_cache_files(self._indices)
+ return [{"filename": cache_filename} for cache_filename in cache_files]
+
+ @property
+ def num_columns(self) -> int:
+ """Number of columns in the dataset.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds.num_columns
+ 2
+ ```
+ """
+ return self._data.num_columns
+
+ @property
+ def num_rows(self) -> int:
+ """Number of rows in the dataset (same as [`Dataset.__len__`]).
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds.num_rows
+ 1066
+ ```
+ """
+ if self._indices is not None:
+ return self._indices.num_rows
+ return self._data.num_rows
+
+ @property
+ def column_names(self) -> list[str]:
+ """Names of the columns in the dataset.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds.column_names
+ ['text', 'label']
+ ```
+ """
+ return self._data.column_names
+
+ @property
+ def shape(self) -> tuple[int, int]:
+ """Shape of the dataset (number of columns, number of rows).
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds.shape
+ (1066, 2)
+ ```
+ """
+ if self._indices is not None:
+ return (self._indices.num_rows, self._data.num_columns)
+ return self._data.shape
+
+ def unique(self, column: str) -> list:
+ """Return a list of the unique elements in a column.
+
+ This is implemented in the low-level backend and as such, very fast.
+
+ Args:
+ column (`str`):
+ Column name (list all the column names with [`~datasets.Dataset.column_names`]).
+
+ Returns:
+ `list`: List of unique elements in the given column.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds.unique('label')
+ [1, 0]
+ ```
+ """
+ if column not in self._data.column_names:
+ raise ValueError(f"Column ({column}) not in table columns ({self._data.column_names}).")
+
+ if self._indices is not None and self._indices.num_rows != self._data.num_rows:
+ dataset = self.flatten_indices()
+ else:
+ dataset = self
+
+ return dataset._data.column(column).unique().to_pylist()
+
+ def class_encode_column(self, column: str, include_nulls: bool = False) -> "Dataset":
+ """Casts the given column as [`~datasets.features.ClassLabel`] and updates the table.
+
+ Args:
+ column (`str`):
+ The name of the column to cast (list all the column names with [`~datasets.Dataset.column_names`])
+ include_nulls (`bool`, defaults to `False`):
+ Whether to include null values in the class labels. If `True`, the null values will be encoded as the `"None"` class label.
+
+
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("google/boolq", split="validation")
+ >>> ds.features
+ {'answer': Value('bool'),
+ 'passage': Value('string'),
+ 'question': Value('string')}
+ >>> ds = ds.class_encode_column('answer')
+ >>> ds.features
+ {'answer': ClassLabel(num_classes=2, names=['False', 'True']),
+ 'passage': Value('string'),
+ 'question': Value('string')}
+ ```
+ """
+ # Sanity checks
+ if column not in self._data.column_names:
+ raise ValueError(f"Column ({column}) not in table columns ({self._data.column_names}).")
+ src_feat = self._info.features[column]
+ if not isinstance(src_feat, Value):
+ raise ValueError(
+ f"Class encoding is only supported for {Value.__name__} column, and column {column} is {type(src_feat).__name__}."
+ )
+
+ if src_feat.dtype != "string" or (include_nulls and None in self.unique(column)):
+
+ def stringify_column(batch):
+ batch[column] = [
+ str(sample) if include_nulls or sample is not None else None for sample in batch[column]
+ ]
+ return batch
+
+ dset = self.map(
+ stringify_column,
+ batched=True,
+ desc="Stringifying the column",
+ )
+ else:
+ dset = self
+
+ # Create the new feature
+ class_names = sorted(str(sample) for sample in dset.unique(column) if include_nulls or sample is not None)
+ dst_feat = ClassLabel(names=class_names)
+
+ def cast_to_class_labels(batch):
+ batch[column] = [
+ dst_feat.str2int(str(sample)) if include_nulls or sample is not None else None
+ for sample in batch[column]
+ ]
+ return batch
+
+ new_features = dset.features.copy()
+ new_features[column] = dst_feat
+
+ dset = dset.map(
+ cast_to_class_labels,
+ batched=True,
+ features=new_features,
+ desc="Casting to class labels",
+ )
+
+ return dset
+
+ @fingerprint_transform(inplace=False)
+ def flatten(self, new_fingerprint: Optional[str] = None, max_depth=16) -> "Dataset":
+ """Flatten the table.
+ Each column with a struct type is flattened into one column per struct field.
+ Other columns are left unchanged.
+
+ Args:
+ new_fingerprint (`str`, *optional*):
+ The new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments.
+
+ Returns:
+ [`Dataset`]: A copy of the dataset with flattened columns.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("rajpurkar/squad", split="train")
+ >>> ds.features
+ {'id': Value('string'),
+ 'title': Value('string'),
+ 'context': Value('string'),
+ 'question': Value('string'),
+ 'answers': {'text': List(Value('string')),
+ 'answer_start': List(Value('int32'))}}
+ >>> ds = ds.flatten()
+ >>> ds
+ Dataset({
+ features: ['id', 'title', 'context', 'question', 'answers.text', 'answers.answer_start'],
+ num_rows: 87599
+ })
+ ```
+ """
+ dataset = copy.deepcopy(self)
+ for depth in range(1, max_depth):
+ if any(isinstance(field.type, pa.StructType) for field in dataset._data.schema):
+ dataset._data = dataset._data.flatten()
+ else:
+ break
+ dataset.info.features = self._info.features.flatten(max_depth=max_depth)
+ dataset.info.features = Features({col: dataset.info.features[col] for col in dataset.data.column_names})
+ dataset._data = update_metadata_with_features(dataset._data, dataset.features)
+ logger.info(f"Flattened dataset from depth {depth} to depth {1 if depth + 1 < max_depth else 'unknown'}.")
+ dataset._fingerprint = new_fingerprint
+ return dataset
+
+ def cast(
+ self,
+ features: Features,
+ batch_size: Optional[int] = 1000,
+ keep_in_memory: bool = False,
+ load_from_cache_file: Optional[bool] = None,
+ cache_file_name: Optional[str] = None,
+ writer_batch_size: Optional[int] = 1000,
+ num_proc: Optional[int] = None,
+ ) -> "Dataset":
+ """
+ Cast the dataset to a new set of features.
+
+ Args:
+ features ([`Features`]):
+ New features to cast the dataset to.
+ The name of the fields in the features must match the current column names.
+ The type of the data must also be convertible from one type to the other.
+ For non-trivial conversion, e.g. `str` <-> `ClassLabel` you should use [`~datasets.Dataset.map`] to update the Dataset.
+ batch_size (`int`, defaults to `1000`):
+ Number of examples per batch provided to cast.
+ If `batch_size <= 0` or `batch_size == None` then provide the full dataset as a single batch to cast.
+ keep_in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+ load_from_cache_file (`bool`, defaults to `True` if caching is enabled):
+ If a cache file storing the current computation from `function`
+ can be identified, use it instead of recomputing.
+ cache_file_name (`str`, *optional*, defaults to `None`):
+ Provide the name of a path for the cache file. It is used to store the
+ results of the computation instead of the automatically generated cache file name.
+ writer_batch_size (`int`, defaults to `1000`):
+ Number of rows per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running [`~datasets.Dataset.map`].
+ num_proc (`int`, *optional*, defaults to `None`):
+ Number of processes for multiprocessing. By default it doesn't
+ use multiprocessing.
+
+ Returns:
+ [`Dataset`]: A copy of the dataset with casted features.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset, ClassLabel, Value
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds.features
+ {'label': ClassLabel(names=['neg', 'pos']),
+ 'text': Value('string')}
+ >>> new_features = ds.features.copy()
+ >>> new_features['label'] = ClassLabel(names=['bad', 'good'])
+ >>> new_features['text'] = Value('large_string')
+ >>> ds = ds.cast(new_features)
+ >>> ds.features
+ {'label': ClassLabel(names=['bad', 'good']),
+ 'text': Value('large_string')}
+ ```
+ """
+ if sorted(features) != sorted(self._data.column_names):
+ raise ValueError(
+ f"The columns in features ({list(features)}) must be identical "
+ f"as the columns in the dataset: {self._data.column_names}"
+ )
+
+ features = _fix_for_backward_compatible_features(features)
+ schema = features.arrow_schema
+ format = self.format
+ dataset = self.with_format("arrow")
+ # capture the PyArrow version here to make the lambda serializable on Windows
+ dataset = dataset.map(
+ partial(table_cast, schema=schema),
+ batched=True,
+ batch_size=batch_size,
+ keep_in_memory=keep_in_memory,
+ load_from_cache_file=load_from_cache_file,
+ cache_file_name=cache_file_name,
+ writer_batch_size=writer_batch_size,
+ num_proc=num_proc,
+ features=features,
+ desc="Casting the dataset",
+ )
+ dataset = dataset.with_format(**format)
+ return dataset
+
+ @fingerprint_transform(inplace=False)
+ def cast_column(self, column: str, feature: FeatureType, new_fingerprint: Optional[str] = None) -> "Dataset":
+ """Cast column to feature for decoding.
+
+ Args:
+ column (`str`):
+ Column name.
+ feature (`FeatureType`):
+ Target feature.
+ new_fingerprint (`str`, *optional*):
+ The new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments.
+
+ Returns:
+ [`Dataset`]
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset, ClassLabel
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds.features
+ {'label': ClassLabel(names=['neg', 'pos']),
+ 'text': Value('string')}
+ >>> ds = ds.cast_column('label', ClassLabel(names=['bad', 'good']))
+ >>> ds.features
+ {'label': ClassLabel(names=['bad', 'good']),
+ 'text': Value('string')}
+ ```
+ """
+ feature = _fix_for_backward_compatible_features(feature)
+ if hasattr(feature, "decode_example"):
+ dataset = copy.deepcopy(self)
+ dataset._info.features[column] = feature
+ dataset._fingerprint = new_fingerprint
+ dataset._data = dataset._data.cast(dataset.features.arrow_schema)
+ dataset._data = update_metadata_with_features(dataset._data, dataset.features)
+ return dataset
+ else:
+ features = self.features
+ features[column] = feature
+ return self.cast(features)
+
+ @transmit_format
+ @fingerprint_transform(inplace=False)
+ def remove_columns(self, column_names: Union[str, list[str]], new_fingerprint: Optional[str] = None) -> "Dataset":
+ """
+ Remove one or several column(s) in the dataset and the features associated to them.
+
+ You can also remove a column using [`~datasets.Dataset.map`] with `remove_columns` but the present method
+ doesn't copy the data of the remaining columns and is thus faster.
+
+ Args:
+ column_names (`Union[str, List[str]]`):
+ Name of the column(s) to remove.
+ new_fingerprint (`str`, *optional*):
+ The new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments.
+
+ Returns:
+ [`Dataset`]: A copy of the dataset object without the columns to remove.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds = ds.remove_columns('label')
+ Dataset({
+ features: ['text'],
+ num_rows: 1066
+ })
+ >>> ds = ds.remove_columns(column_names=ds.column_names) # Removing all the columns returns an empty dataset with the `num_rows` property set to 0
+ Dataset({
+ features: [],
+ num_rows: 0
+ })
+ ```
+ """
+ dataset = copy.deepcopy(self)
+ if isinstance(column_names, str):
+ column_names = [column_names]
+
+ missing_columns = set(column_names) - set(self._data.column_names)
+ if missing_columns:
+ raise ValueError(
+ f"Column name {list(missing_columns)} not in the dataset. "
+ f"Current columns in the dataset: {dataset._data.column_names}"
+ )
+
+ for column_name in column_names:
+ del dataset._info.features[column_name]
+
+ dataset._data = dataset._data.drop(column_names)
+ dataset._data = update_metadata_with_features(dataset._data, dataset.features)
+ dataset._fingerprint = new_fingerprint
+ return dataset
+
+ @fingerprint_transform(inplace=False)
+ def rename_column(
+ self, original_column_name: str, new_column_name: str, new_fingerprint: Optional[str] = None
+ ) -> "Dataset":
+ """
+ Rename a column in the dataset, and move the features associated to the original column under the new column
+ name.
+
+ Args:
+ original_column_name (`str`):
+ Name of the column to rename.
+ new_column_name (`str`):
+ New name for the column.
+ new_fingerprint (`str`, *optional*):
+ The new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments.
+
+ Returns:
+ [`Dataset`]: A copy of the dataset with a renamed column.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds = ds.rename_column('label', 'label_new')
+ Dataset({
+ features: ['text', 'label_new'],
+ num_rows: 1066
+ })
+ ```
+ """
+ dataset = copy.deepcopy(self)
+ if original_column_name not in dataset._data.column_names:
+ raise ValueError(
+ f"Original column name {original_column_name} not in the dataset. "
+ f"Current columns in the dataset: {dataset._data.column_names}"
+ )
+ if new_column_name in dataset._data.column_names:
+ raise ValueError(
+ f"New column name {new_column_name} already in the dataset. "
+ f"Please choose a column name which is not already in the dataset. "
+ f"Current columns in the dataset: {dataset._data.column_names}"
+ )
+ if not new_column_name:
+ raise ValueError("New column name is empty.")
+
+ def rename(columns):
+ return [new_column_name if col == original_column_name else col for col in columns]
+
+ new_column_names = rename(self._data.column_names)
+ if self._format_columns is not None:
+ dataset._format_columns = rename(self._format_columns)
+
+ dataset._info.features = Features(
+ {
+ new_column_name if col == original_column_name else col: feature
+ for col, feature in self._info.features.items()
+ }
+ )
+
+ dataset._data = dataset._data.rename_columns(new_column_names)
+ dataset._data = update_metadata_with_features(dataset._data, dataset.features)
+ dataset._fingerprint = new_fingerprint
+ return dataset
+
+ @fingerprint_transform(inplace=False)
+ def rename_columns(self, column_mapping: dict[str, str], new_fingerprint: Optional[str] = None) -> "Dataset":
+ """
+ Rename several columns in the dataset, and move the features associated to the original columns under
+ the new column names.
+
+ Args:
+ column_mapping (`Dict[str, str]`):
+ A mapping of columns to rename to their new names
+ new_fingerprint (`str`, *optional*):
+ The new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments.
+
+ Returns:
+ [`Dataset`]: A copy of the dataset with renamed columns
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds = ds.rename_columns({'text': 'text_new', 'label': 'label_new'})
+ Dataset({
+ features: ['text_new', 'label_new'],
+ num_rows: 1066
+ })
+ ```
+ """
+ dataset = copy.deepcopy(self)
+
+ extra_columns = set(column_mapping.keys()) - set(dataset.column_names)
+ if extra_columns:
+ raise ValueError(
+ f"Original column names {extra_columns} not in the dataset. "
+ f"Current columns in the dataset: {dataset._data.column_names}"
+ )
+
+ number_of_duplicates_in_new_columns = len(column_mapping.values()) - len(set(column_mapping.values()))
+ if number_of_duplicates_in_new_columns != 0:
+ raise ValueError(
+ "New column names must all be different, but this column mapping "
+ f"has {number_of_duplicates_in_new_columns} duplicates"
+ )
+
+ empty_new_columns = [new_col for new_col in column_mapping.values() if not new_col]
+ if empty_new_columns:
+ raise ValueError(f"New column names {empty_new_columns} are empty.")
+
+ def rename(columns):
+ return [column_mapping[col] if col in column_mapping else col for col in columns]
+
+ new_column_names = rename(self._data.column_names)
+ if self._format_columns is not None:
+ dataset._format_columns = rename(self._format_columns)
+
+ dataset._info.features = Features(
+ {
+ column_mapping[col] if col in column_mapping else col: feature
+ for col, feature in (self._info.features or {}).items()
+ }
+ )
+
+ dataset._data = dataset._data.rename_columns(new_column_names)
+ dataset._data = update_metadata_with_features(dataset._data, dataset.features)
+ dataset._fingerprint = new_fingerprint
+ return dataset
+
+ @transmit_format
+ @fingerprint_transform(inplace=False)
+ def select_columns(self, column_names: Union[str, list[str]], new_fingerprint: Optional[str] = None) -> "Dataset":
+ """Select one or several column(s) in the dataset and the features
+ associated to them.
+
+ Args:
+ column_names (`Union[str, List[str]]`):
+ Name of the column(s) to keep.
+ new_fingerprint (`str`, *optional*):
+ The new fingerprint of the dataset after transform. If `None`,
+ the new fingerprint is computed using a hash of the previous
+ fingerprint, and the transform arguments.
+
+ Returns:
+ [`Dataset`]: A copy of the dataset object which only consists of
+ selected columns.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds = ds.select_columns(['text'])
+ >>> ds
+ Dataset({
+ features: ['text'],
+ num_rows: 1066
+ })
+ ```
+ """
+ if isinstance(column_names, str):
+ column_names = [column_names]
+
+ missing_columns = set(column_names) - set(self._data.column_names)
+ if missing_columns:
+ raise ValueError(
+ f"Column name {list(missing_columns)} not in the "
+ "dataset. Current columns in the dataset: "
+ f"{self._data.column_names}."
+ )
+
+ dataset = copy.deepcopy(self)
+ dataset._data = dataset._data.select(column_names)
+ dataset._info.features = Features({col: self._info.features[col] for col in dataset._data.column_names})
+ dataset._data = update_metadata_with_features(dataset._data, dataset.features)
+ dataset._fingerprint = new_fingerprint
+ return dataset
+
+ @transmit_format
+ def _fast_select_column(self, column_name: str) -> "Dataset":
+ dataset = copy.copy(self)
+ dataset._data = dataset._data.select([column_name])
+ dataset._info = DatasetInfo(features=Features({column_name: self._info.features[column_name]}))
+ return dataset
+
+ def __len__(self):
+ """Number of rows in the dataset.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds.__len__
+
+ ```
+ """
+ return self.num_rows
+
+ def __iter__(self):
+ """Iterate through the examples.
+
+ If a formatting is set with [`Dataset.set_format`] rows will be returned with the
+ selected format.
+ """
+ if self._indices is None:
+ # Fast iteration
+ # Benchmark: https://gist.github.com/mariosasko/0248288a2e3a7556873969717c1fe52b (fast_iter_batch)
+ format_kwargs = self._format_kwargs if self._format_kwargs is not None else {}
+ formatter = get_formatter(self._format_type, features=self._info.features, **format_kwargs)
+ batch_size = config.ARROW_READER_BATCH_SIZE_IN_DATASET_ITER
+ for pa_subtable in table_iter(self.data, batch_size=batch_size):
+ for i in range(pa_subtable.num_rows):
+ pa_subtable_ex = pa_subtable.slice(i, 1)
+ formatted_output = format_table(
+ pa_subtable_ex,
+ 0,
+ formatter=formatter,
+ format_columns=self._format_columns,
+ output_all_columns=self._output_all_columns,
+ )
+ yield formatted_output
+ else:
+ for i in range(self.num_rows):
+ yield self._getitem(
+ i,
+ )
+
+ def iter(self, batch_size: int, drop_last_batch: bool = False):
+ """Iterate through the batches of size `batch_size`.
+
+ If a formatting is set with [`~datasets.Dataset.set_format`] rows will be returned with the
+ selected format.
+
+ Args:
+ batch_size (:obj:`int`): size of each batch to yield.
+ drop_last_batch (:obj:`bool`, default `False`): Whether a last batch smaller than the batch_size should be
+ dropped
+ """
+ if self._indices is None:
+ # Fast iteration
+ # Benchmark: https://gist.github.com/mariosasko/0248288a2e3a7556873969717c1fe52b (fast_iter_batch)
+ format_kwargs = self._format_kwargs if self._format_kwargs is not None else {}
+ formatter = get_formatter(self._format_type, features=self._info.features, **format_kwargs)
+ for pa_subtable in table_iter(self.data, batch_size=batch_size, drop_last_batch=drop_last_batch):
+ formatted_batch = format_table(
+ pa_subtable,
+ range(pa_subtable.num_rows),
+ formatter=formatter,
+ format_columns=self._format_columns,
+ output_all_columns=self._output_all_columns,
+ )
+ yield formatted_batch
+ else:
+ num_rows = self.num_rows if not drop_last_batch else self.num_rows // batch_size * batch_size
+ for i in range(0, num_rows, batch_size):
+ yield self._getitem(
+ slice(i, i + batch_size),
+ )
+
+ def __repr__(self):
+ return f"Dataset({{\n features: {list(self._info.features.keys())},\n num_rows: {self.num_rows}\n}})"
+
+ @property
+ def format(self):
+ return {
+ "type": self._format_type,
+ "format_kwargs": self._format_kwargs,
+ "columns": self.column_names if self._format_columns is None else self._format_columns,
+ "output_all_columns": self._output_all_columns,
+ }
+
+ @contextlib.contextmanager
+ def formatted_as(
+ self,
+ type: Optional[str] = None,
+ columns: Optional[list] = None,
+ output_all_columns: bool = False,
+ **format_kwargs,
+ ):
+ """To be used in a `with` statement. Set `__getitem__` return format (type and columns).
+
+ Args:
+ type (`str`, *optional*):
+ Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`.
+ `None` means `__getitem__`` returns python objects (default).
+ columns (`List[str]`, *optional*):
+ Columns to format in the output.
+ `None` means `__getitem__` returns all columns (default).
+ output_all_columns (`bool`, defaults to `False`):
+ Keep un-formatted columns as well in the output (as python objects).
+ **format_kwargs (additional keyword arguments):
+ Keywords arguments passed to the convert function like `np.array`, `torch.tensor` or `tensorflow.ragged.constant`.
+ """
+ old_format_type = self._format_type
+ old_format_kwargs = self._format_kwargs
+ old_format_columns = self._format_columns
+ old_output_all_columns = self._output_all_columns
+ try:
+ self.set_format(type, columns, output_all_columns, **format_kwargs)
+ yield
+ finally:
+ self.set_format(old_format_type, old_format_columns, old_output_all_columns, **old_format_kwargs)
+
+ @fingerprint_transform(inplace=True)
+ def set_format(
+ self,
+ type: Optional[str] = None,
+ columns: Optional[list] = None,
+ output_all_columns: bool = False,
+ **format_kwargs,
+ ):
+ """Set `__getitem__` return format (type and columns). The data formatting is applied on-the-fly.
+ The format `type` (for example "numpy") is used to format batches when using `__getitem__`.
+ It's also possible to use custom transforms for formatting using [`~datasets.Dataset.set_transform`].
+
+ Args:
+ type (`str`, *optional*):
+ Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`.
+ `None` means `__getitem__` returns python objects (default).
+ columns (`List[str]`, *optional*):
+ Columns to format in the output.
+ `None` means `__getitem__` returns all columns (default).
+ output_all_columns (`bool`, defaults to `False`):
+ Keep un-formatted columns as well in the output (as python objects).
+ **format_kwargs (additional keyword arguments):
+ Keywords arguments passed to the convert function like `np.array`, `torch.tensor` or `tensorflow.ragged.constant`.
+
+ It is possible to call [`~datasets.Dataset.map`] after calling `set_format`. Since `map` may add new columns, then the list of formatted columns
+ gets updated. In this case, if you apply `map` on a dataset to add a new column, then this column will be formatted as:
+
+ ```
+ new formatted columns = (all columns - previously unformatted columns)
+ ```
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> from transformers import AutoTokenizer
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+ >>> ds = ds.map(lambda x: tokenizer(x['text'], truncation=True, padding=True), batched=True)
+ >>> ds.set_format(type='numpy', columns=['text', 'label'])
+ >>> ds.format
+ {'type': 'numpy',
+ 'format_kwargs': {},
+ 'columns': ['text', 'label'],
+ 'output_all_columns': False}
+ ```
+ """
+ format_kwargs.update(format_kwargs.pop("format_kwargs", {})) # allow to use self.set_format(**self.format)
+
+ # Check that the format_type and format_kwargs are valid and make it possible to have a Formatter
+ type = get_format_type_from_alias(type)
+ get_formatter(type, features=self._info.features, **format_kwargs)
+
+ # Check filter column
+ if isinstance(columns, str):
+ columns = [columns]
+ if isinstance(columns, tuple):
+ columns = list(columns)
+ if columns is not None:
+ missing_columns = set(columns) - set(self._data.column_names)
+ if missing_columns:
+ raise ValueError(
+ f"Columns {list(missing_columns)} not in the dataset. Current columns in the dataset: {self._data.column_names}"
+ )
+ if columns is not None:
+ columns = columns.copy() # Ensures modifications made to the list after this call don't cause bugs
+
+ self._format_type = type
+ self._format_kwargs = format_kwargs
+ self._format_columns = columns
+ self._output_all_columns = output_all_columns
+ logger.debug(
+ "Set __getitem__(key) output type to %s for %s columns "
+ " (when key is int or slice) and %s output other (un-formatted) columns.",
+ "python objects" if type is None else type,
+ "no" if columns is None else str(columns),
+ "do" if output_all_columns else "don't",
+ )
+
+ def reset_format(self):
+ """Reset `__getitem__` return format to python objects and all columns.
+
+ Same as `self.set_format()`
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> from transformers import AutoTokenizer
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+ >>> ds = ds.map(lambda x: tokenizer(x['text'], truncation=True, padding=True), batched=True)
+ >>> ds.set_format(type='numpy', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])
+ >>> ds.format
+ {'columns': ['input_ids', 'token_type_ids', 'attention_mask', 'label'],
+ 'format_kwargs': {},
+ 'output_all_columns': False,
+ 'type': 'numpy'}
+ >>> ds.reset_format()
+ >>> ds.format
+ {'columns': ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'],
+ 'format_kwargs': {},
+ 'output_all_columns': False,
+ 'type': None}
+ ```
+ """
+ self.set_format()
+
+ def set_transform(
+ self,
+ transform: Optional[Callable],
+ columns: Optional[list] = None,
+ output_all_columns: bool = False,
+ ):
+ """Set `__getitem__` return format using this transform. The transform is applied on-the-fly on batches when `__getitem__` is called.
+ As [`~datasets.Dataset.set_format`], this can be reset using [`~datasets.Dataset.reset_format`].
+
+ Args:
+ transform (`Callable`, *optional*):
+ User-defined formatting transform, replaces the format defined by [`~datasets.Dataset.set_format`].
+ A formatting function is a callable that takes a batch (as a `dict`) as input and returns a batch.
+ This function is applied right before returning the objects in `__getitem__`.
+ columns (`List[str]`, *optional*):
+ Columns to format in the output.
+ If specified, then the input batch of the transform only contains those columns.
+ output_all_columns (`bool`, defaults to `False`):
+ Keep un-formatted columns as well in the output (as python objects).
+ If set to True, then the other un-formatted columns are kept with the output of the transform.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> from transformers import AutoTokenizer
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
+ >>> def encode(batch):
+ ... return tokenizer(batch['text'], padding=True, truncation=True, return_tensors='pt')
+ >>> ds.set_transform(encode)
+ >>> ds[0]
+ {'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1]),
+ 'input_ids': tensor([ 101, 29353, 2135, 15102, 1996, 9428, 20868, 2890, 8663, 6895,
+ 20470, 2571, 3663, 2090, 4603, 3017, 3008, 1998, 2037, 24211,
+ 5637, 1998, 11690, 2336, 1012, 102]),
+ 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0])}
+ ```
+ """
+ self.set_format("custom", columns=columns, output_all_columns=output_all_columns, transform=transform)
+
+ def with_format(
+ self,
+ type: Optional[str] = None,
+ columns: Optional[list] = None,
+ output_all_columns: bool = False,
+ **format_kwargs,
+ ):
+ """Set `__getitem__` return format (type and columns). The data formatting is applied on-the-fly.
+ The format `type` (for example "numpy") is used to format batches when using `__getitem__`.
+
+ It's also possible to use custom transforms for formatting using [`~datasets.Dataset.with_transform`].
+
+ Contrary to [`~datasets.Dataset.set_format`], `with_format` returns a new [`Dataset`] object.
+
+ Args:
+ type (`str`, *optional*):
+ Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`.
+ `None` means `__getitem__` returns python objects (default).
+ columns (`List[str]`, *optional*):
+ Columns to format in the output.
+ `None` means `__getitem__` returns all columns (default).
+ output_all_columns (`bool`, defaults to `False`):
+ Keep un-formatted columns as well in the output (as python objects).
+ **format_kwargs (additional keyword arguments):
+ Keywords arguments passed to the convert function like `np.array`, `torch.tensor` or `tensorflow.ragged.constant`.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> from transformers import AutoTokenizer
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+ >>> ds = ds.map(lambda x: tokenizer(x['text'], truncation=True, padding=True), batched=True)
+ >>> ds.format
+ {'columns': ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'],
+ 'format_kwargs': {},
+ 'output_all_columns': False,
+ 'type': None}
+ >>> ds = ds.with_format("torch")
+ >>> ds.format
+ {'columns': ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'],
+ 'format_kwargs': {},
+ 'output_all_columns': False,
+ 'type': 'torch'}
+ >>> ds[0]
+ {'text': 'compassionately explores the seemingly irreconcilable situation between conservative christian parents and their estranged gay and lesbian children .',
+ 'label': tensor(1),
+ 'input_ids': tensor([ 101, 18027, 16310, 16001, 1103, 9321, 178, 11604, 7235, 6617,
+ 1742, 2165, 2820, 1206, 6588, 22572, 12937, 1811, 2153, 1105,
+ 1147, 12890, 19587, 6463, 1105, 15026, 1482, 119, 102, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0]),
+ 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
+ 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])}
+ ```
+ """
+ dataset = copy.deepcopy(self)
+ dataset.set_format(type=type, columns=columns, output_all_columns=output_all_columns, **format_kwargs)
+ return dataset
+
+ def with_transform(
+ self,
+ transform: Optional[Callable],
+ columns: Optional[list] = None,
+ output_all_columns: bool = False,
+ ):
+ """Set `__getitem__` return format using this transform. The transform is applied on-the-fly on batches when `__getitem__` is called.
+
+ As [`~datasets.Dataset.set_format`], this can be reset using [`~datasets.Dataset.reset_format`].
+
+ Contrary to [`~datasets.Dataset.set_transform`], `with_transform` returns a new [`Dataset`] object.
+
+ Args:
+ transform (`Callable`, `optional`):
+ User-defined formatting transform, replaces the format defined by [`~datasets.Dataset.set_format`].
+ A formatting function is a callable that takes a batch (as a `dict`) as input and returns a batch.
+ This function is applied right before returning the objects in `__getitem__`.
+ columns (`List[str]`, `optional`):
+ Columns to format in the output.
+ If specified, then the input batch of the transform only contains those columns.
+ output_all_columns (`bool`, defaults to `False`):
+ Keep un-formatted columns as well in the output (as python objects).
+ If set to `True`, then the other un-formatted columns are kept with the output of the transform.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> from transformers import AutoTokenizer
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+ >>> def encode(example):
+ ... return tokenizer(example["text"], padding=True, truncation=True, return_tensors='pt')
+ >>> ds = ds.with_transform(encode)
+ >>> ds[0]
+ {'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1]),
+ 'input_ids': tensor([ 101, 18027, 16310, 16001, 1103, 9321, 178, 11604, 7235, 6617,
+ 1742, 2165, 2820, 1206, 6588, 22572, 12937, 1811, 2153, 1105,
+ 1147, 12890, 19587, 6463, 1105, 15026, 1482, 119, 102]),
+ 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0])}
+ ```
+ """
+ dataset = copy.deepcopy(self)
+ dataset.set_transform(transform=transform, columns=columns, output_all_columns=output_all_columns)
+ return dataset
+
+ def _getitem(self, key: Union[int, slice, str, ListLike[int]], **kwargs) -> Union[dict, list]:
+ """
+ Can be used to index columns (by string names) or rows (by integer, slice, or list-like of integer indices)
+ """
+ if isinstance(key, bool):
+ raise TypeError("dataset index must be int, str, slice or collection of int, not bool")
+ format_type = kwargs["format_type"] if "format_type" in kwargs else self._format_type
+ format_columns = kwargs["format_columns"] if "format_columns" in kwargs else self._format_columns
+ output_all_columns = (
+ kwargs["output_all_columns"] if "output_all_columns" in kwargs else self._output_all_columns
+ )
+ format_kwargs = kwargs["format_kwargs"] if "format_kwargs" in kwargs else self._format_kwargs
+ format_kwargs = format_kwargs if format_kwargs is not None else {}
+ formatter = get_formatter(format_type, features=self._info.features, **format_kwargs)
+ pa_subtable = query_table(self._data, key, indices=self._indices)
+ formatted_output = format_table(
+ pa_subtable, key, formatter=formatter, format_columns=format_columns, output_all_columns=output_all_columns
+ )
+ return formatted_output
+
+ @overload
+ def __getitem__(self, key: Union[int, slice, Iterable[int]]) -> dict: # noqa: F811
+ ...
+
+ @overload
+ def __getitem__(self, key: str) -> list: # noqa: F811
+ ...
+
+ def __getitem__(self, key): # noqa: F811
+ """Can be used to index columns (by string names) or rows (by integer index or iterable of indices or bools)."""
+ if isinstance(key, str):
+ if self._format_type is None or self._format_type not in ("arrow", "pandas", "polars"):
+ return Column(self, key)
+ return self._getitem(key)
+
+ def __getitems__(self, keys: list) -> list:
+ """Can be used to get a batch using a list of integers indices."""
+ batch = self.__getitem__(keys)
+ n_examples = len(batch[next(iter(batch))])
+ return [{col: array[i] for col, array in batch.items()} for i in range(n_examples)]
+
+ def cleanup_cache_files(self) -> int:
+ """Clean up all cache files in the dataset cache directory, excepted the currently used cache file if there is
+ one.
+
+ Be careful when running this command that no other process is currently using other cache files.
+
+ Returns:
+ `int`: Number of removed files.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds.cleanup_cache_files()
+ 10
+ ```
+ """
+ current_cache_files = [os.path.abspath(cache_file["filename"]) for cache_file in self.cache_files]
+ if not current_cache_files:
+ return 0
+ cache_directory = os.path.dirname(current_cache_files[0])
+ logger.info(f"Listing files in {cache_directory}")
+ files: list[str] = os.listdir(cache_directory)
+ files_to_remove = []
+ for f_name in files:
+ full_name = os.path.abspath(os.path.join(cache_directory, f_name))
+ if f_name.startswith("cache-") and f_name.endswith(".arrow"):
+ if full_name in current_cache_files:
+ logger.info(f"Keeping currently used cache file at {full_name}")
+ continue
+ files_to_remove.append(full_name)
+ for file_path in files_to_remove:
+ logger.info(f"Removing {file_path}")
+ os.remove(file_path)
+ return len(files_to_remove)
+
+ def _get_cache_file_path(self, fingerprint):
+ if is_caching_enabled() and self.cache_files:
+ cache_file_name = "cache-" + fingerprint + ".arrow"
+ cache_directory = os.path.dirname(self.cache_files[0]["filename"])
+ else:
+ cache_file_name = "cache-" + generate_random_fingerprint() + ".arrow"
+ cache_directory = get_temporary_cache_files_directory()
+ cache_file_path = os.path.join(cache_directory, cache_file_name)
+ return cache_file_path
+
+ @transmit_format
+ def map(
+ self,
+ function: Optional[Callable] = None,
+ with_indices: bool = False,
+ with_rank: bool = False,
+ input_columns: Optional[Union[str, list[str]]] = None,
+ batched: bool = False,
+ batch_size: Optional[int] = 1000,
+ drop_last_batch: bool = False,
+ remove_columns: Optional[Union[str, list[str]]] = None,
+ keep_in_memory: bool = False,
+ load_from_cache_file: Optional[bool] = None,
+ cache_file_name: Optional[str] = None,
+ writer_batch_size: Optional[int] = 1000,
+ features: Optional[Features] = None,
+ disable_nullable: bool = False,
+ fn_kwargs: Optional[dict] = None,
+ num_proc: Optional[int] = None,
+ suffix_template: str = "_{rank:05d}_of_{num_proc:05d}",
+ new_fingerprint: Optional[str] = None,
+ desc: Optional[str] = None,
+ try_original_type: Optional[bool] = True,
+ on_mixed_types: Optional[Literal["use_json"]] = "use_json",
+ ) -> "Dataset":
+ """
+ Apply a function to all the examples in the table (individually or in batches) and update the table.
+ If your function returns a column that already exists, then it overwrites it.
+
+ You can specify whether the function should be batched or not with the `batched` parameter:
+
+ - If batched is `False`, then the function takes 1 example in and should return 1 example.
+ An example is a dictionary, e.g. `{"text": "Hello there !"}`.
+ - If batched is `True` and `batch_size` is 1, then the function takes a batch of 1 example as input and can return a batch with 1 or more examples.
+ A batch is a dictionary, e.g. a batch of 1 example is `{"text": ["Hello there !"]}`.
+ - If batched is `True` and `batch_size` is `n > 1`, then the function takes a batch of `n` examples as input and can return a batch with `n` examples, or with an arbitrary number of examples.
+ Note that the last batch may have less than `n` examples.
+ A batch is a dictionary, e.g. a batch of `n` examples is `{"text": ["Hello there !"] * n}`.
+
+ If the function is asynchronous, then `map` will run your function in parallel, with up to one thousand simultaneous calls.
+ It is recommended to use a `asyncio.Semaphore` in your function if you want to set a maximum number of operations that can run at the same time.
+
+ Args:
+ function (`Callable`): Function with one of the following signatures:
+
+ - `function(example: Dict[str, Any]) -> Dict[str, Any]` if `batched=False` and `with_indices=False` and `with_rank=False`
+ - `function(example: Dict[str, Any], *extra_args) -> Dict[str, Any]` if `batched=False` and `with_indices=True` and/or `with_rank=True` (one extra arg for each)
+ - `function(batch: Dict[str, List]) -> Dict[str, List]` if `batched=True` and `with_indices=False` and `with_rank=False`
+ - `function(batch: Dict[str, List], *extra_args) -> Dict[str, List]` if `batched=True` and `with_indices=True` and/or `with_rank=True` (one extra arg for each)
+
+ For advanced usage, the function can also return a `pyarrow.Table`.
+ If the function is asynchronous, then `map` will run your function in parallel.
+ Moreover if your function returns nothing (`None`), then `map` will run your function and return the dataset unchanged.
+ If no function is provided, default to identity function: `lambda x: x`.
+ with_indices (`bool`, defaults to `False`):
+ Provide example indices to `function`. Note that in this case the
+ signature of `function` should be `def function(example, idx[, rank]): ...`.
+ with_rank (`bool`, defaults to `False`):
+ Provide process rank to `function`. Note that in this case the
+ signature of `function` should be `def function(example[, idx], rank): ...`.
+ input_columns (`Optional[Union[str, List[str]]]`, defaults to `None`):
+ The columns to be passed into `function`
+ as positional arguments. If `None`, a `dict` mapping to all formatted columns is passed as one argument.
+ batched (`bool`, defaults to `False`):
+ Provide batch of examples to `function`.
+ batch_size (`int`, *optional*, defaults to `1000`):
+ Number of examples per batch provided to `function` if `batched=True`.
+ If `batch_size <= 0` or `batch_size == None`, provide the full dataset as a single batch to `function`.
+ drop_last_batch (`bool`, defaults to `False`):
+ Whether a last batch smaller than the batch_size should be
+ dropped instead of being processed by the function.
+ remove_columns (`Optional[Union[str, List[str]]]`, defaults to `None`):
+ Remove a selection of columns while doing the mapping.
+ Columns will be removed before updating the examples with the output of `function`, i.e. if `function` is adding
+ columns with names in `remove_columns`, these columns will be kept.
+ keep_in_memory (`bool`, defaults to `False`):
+ Keep the dataset in memory instead of writing it to a cache file.
+ load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled):
+ If a cache file storing the current computation from `function`
+ can be identified, use it instead of recomputing.
+ cache_file_name (`str`, *optional*, defaults to `None`):
+ Provide the name of a path for the cache file. It is used to store the
+ results of the computation instead of the automatically generated cache file name.
+ writer_batch_size (`int`, defaults to `1000`):
+ Number of rows per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`.
+ features (`Optional[datasets.Features]`, defaults to `None`):
+ Use a specific Features to store the cache file
+ instead of the automatically generated one.
+ disable_nullable (`bool`, defaults to `False`):
+ Disallow null values in the table.
+ fn_kwargs (`Dict`, *optional*, defaults to `None`):
+ Keyword arguments to be passed to `function`.
+ num_proc (`int`, *optional*, defaults to `None`):
+ The number of processes to use for multiprocessing.
+ - If `None` or `0`, no multiprocessing is used and the operation runs in the main process.
+ - If greater than `1`, one or multiple worker processes are used to process data in parallel.
+ Note: The function passed to `map()` must be picklable for multiprocessing to work correctly
+ (i.e., prefer functions defined at the top level of a module, not inside another function or class).
+ suffix_template (`str`):
+ If `cache_file_name` is specified, then this suffix
+ will be added at the end of the base name of each. Defaults to `"_{rank:05d}_of_{num_proc:05d}"`. For example, if `cache_file_name` is "processed.arrow", then for
+ `rank=1` and `num_proc=4`, the resulting file would be `"processed_00001_of_00004.arrow"` for the default suffix.
+ new_fingerprint (`str`, *optional*, defaults to `None`):
+ The new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments.
+ desc (`str`, *optional*, defaults to `None`):
+ Meaningful description to be displayed alongside with the progress bar while mapping examples.
+ try_original_type (`Optional[bool]`, defaults to `True`):
+ Try to keep the types of the original columns (e.g. int32 -> int32).
+ Set to False if you want to always infer new types.
+ on_mixed_types (`Literal["use_json"]`, *optional*, defaults to `None`):
+ If "use_json", use the Json() type for mixed-types fields,
+ i.e. unstructured fields that contain data without a predefined schema.
+ In this case, a field with mixed type is set to Json().
+
+ This allow loading lists with a mix of strings/integers/floats
+ for example, or dictionaries with arbitrary value types.
+
+
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> def add_prefix(example):
+ ... example["text"] = "Review: " + example["text"]
+ ... return example
+ >>> ds = ds.map(add_prefix)
+ >>> ds[0:3]["text"]
+ ['Review: compassionately explores the seemingly irreconcilable situation between conservative christian parents and their estranged gay and lesbian children .',
+ 'Review: the soundtrack alone is worth the price of admission .',
+ 'Review: rodriguez does a splendid job of racial profiling hollywood style--casting excellent latin actors of all ages--a trend long overdue .']
+
+ # process a batch of examples
+ >>> ds = ds.map(lambda example: tokenizer(example["text"]), batched=True)
+ # set number of processors
+ >>> ds = ds.map(add_prefix, num_proc=4)
+ ```
+ """
+ if keep_in_memory and cache_file_name is not None:
+ raise ValueError("Please use either `keep_in_memory` or `cache_file_name` but not both.")
+
+ if num_proc == 0:
+ num_proc = None
+ elif num_proc is not None and num_proc < 0:
+ raise ValueError("num_proc must be >= 0 or None.")
+
+ string_formatter = string.Formatter()
+ fields = {field_name for _, field_name, _, _ in string_formatter.parse(suffix_template) if field_name}
+ if fields != {"rank", "num_proc"}:
+ raise ValueError(f"suffix_template must contain exactly the fields 'rank' and 'num_proc', got: {fields}")
+
+ # If the array is empty we do nothing (but we make sure to handle an empty indices mapping and remove the requested columns anyway)
+ if len(self) == 0:
+ if self._indices is not None: # empty indices mapping
+ self = Dataset(
+ self.data.slice(0, 0),
+ info=self.info.copy(),
+ split=self.split,
+ fingerprint=new_fingerprint,
+ )
+ if remove_columns:
+ return self.remove_columns(remove_columns)
+ else:
+ return self
+
+ if function is None:
+ function = lambda x: x # noqa: E731
+
+ if isinstance(input_columns, str):
+ input_columns = [input_columns]
+
+ if input_columns is not None:
+ missing_columns = set(input_columns) - set(self._data.column_names)
+ if missing_columns:
+ raise ValueError(
+ f"Input column {list(missing_columns)} not in the dataset. Current columns in the dataset: {self._data.column_names}"
+ )
+
+ if isinstance(remove_columns, str):
+ remove_columns = [remove_columns]
+
+ if remove_columns is not None:
+ missing_columns = set(remove_columns) - set(self._data.column_names)
+ if missing_columns:
+ raise ValueError(
+ f"Column to remove {list(missing_columns)} not in the dataset. Current columns in the dataset: {self._data.column_names}"
+ )
+
+ load_from_cache_file = load_from_cache_file if load_from_cache_file is not None else is_caching_enabled()
+
+ if fn_kwargs is None:
+ fn_kwargs = {}
+
+ if features is not None:
+ features = _fix_for_backward_compatible_features(features)
+
+ if num_proc is not None and num_proc > len(self):
+ num_proc = len(self)
+ logger.warning(
+ f"num_proc must be <= {len(self)}. Reducing num_proc to {num_proc} for dataset of size {len(self)}."
+ )
+
+ dataset_kwargs = {
+ "shard": self,
+ "function": function,
+ "with_indices": with_indices,
+ "with_rank": with_rank,
+ "input_columns": input_columns,
+ "batched": batched,
+ "batch_size": batch_size,
+ "drop_last_batch": drop_last_batch,
+ "remove_columns": remove_columns,
+ "keep_in_memory": keep_in_memory,
+ "writer_batch_size": writer_batch_size,
+ "features": features,
+ "disable_nullable": disable_nullable,
+ "fn_kwargs": fn_kwargs,
+ "try_original_type": try_original_type,
+ "on_mixed_types": on_mixed_types,
+ }
+
+ if new_fingerprint is None:
+ # we create a unique hash from the function,
+ # current dataset file and the mapping args
+ transform = format_transform_for_fingerprint(Dataset._map_single)
+ kwargs_for_fingerprint = format_kwargs_for_fingerprint(Dataset._map_single, (), dataset_kwargs)
+ kwargs_for_fingerprint["fingerprint_name"] = "new_fingerprint"
+ new_fingerprint = update_fingerprint(self._fingerprint, transform, kwargs_for_fingerprint)
+ else:
+ validate_fingerprint(new_fingerprint)
+ dataset_kwargs["new_fingerprint"] = new_fingerprint
+
+ if self.cache_files:
+ if cache_file_name is None:
+ cache_file_name = self._get_cache_file_path(new_fingerprint)
+ dataset_kwargs["cache_file_name"] = cache_file_name
+
+ if cache_file_name is not None:
+ cache_file_prefix, cache_file_ext = os.path.splitext(cache_file_name)
+ if not cache_file_ext:
+ raise ValueError(f"Expected cache_file_name to have an extension, but got: {cache_file_name}")
+ else:
+ cache_file_prefix = cache_file_ext = None
+
+ def load_processed_shard_from_cache(shard_kwargs: dict[str, Any]) -> Dataset:
+ """Load a processed shard from cache if it exists, otherwise throw an error."""
+ shard = shard_kwargs["shard"]
+ # Check if we've already cached this computation (indexed by a hash)
+ if shard_kwargs["cache_file_name"] is not None:
+ if os.path.exists(shard_kwargs["cache_file_name"]) and load_from_cache_file:
+ info = shard.info.copy()
+ info.features = features
+ return Dataset.from_file(shard_kwargs["cache_file_name"], info=info, split=shard.split)
+ raise NonExistentDatasetError
+
+ existing_cache_file_map: dict[int, list[str]] = defaultdict(list)
+ if cache_file_name is not None:
+ if os.path.exists(cache_file_name):
+ existing_cache_file_map[1] = [cache_file_name]
+
+ assert cache_file_prefix is not None and cache_file_ext is not None
+ cache_file_with_suffix_pattern = cache_file_prefix + suffix_template + cache_file_ext
+
+ for cache_file in glob.iglob(f"{cache_file_prefix}*{cache_file_ext}"):
+ suffix_variable_map = string_to_dict(
+ Path(cache_file).as_posix(), Path(cache_file_with_suffix_pattern).as_posix()
+ )
+ if suffix_variable_map is not None:
+ file_num_proc = int(suffix_variable_map["num_proc"])
+ existing_cache_file_map[file_num_proc].append(cache_file)
+
+ num_shards = num_proc or 1
+ if existing_cache_file_map:
+ # to avoid remapping when a different num_proc is given than when originally cached, update num_shards to
+ # what was used originally
+
+ def select_existing_cache_files(mapped_num_proc: int) -> tuple[float, ...]:
+ percent_missing = (mapped_num_proc - len(existing_cache_file_map[mapped_num_proc])) / mapped_num_proc
+ num_shards_diff = abs(mapped_num_proc - num_shards)
+ return (
+ percent_missing, # choose the most complete set of existing cache files
+ num_shards_diff, # then choose the mapped_num_proc closest to the current num_proc
+ mapped_num_proc, # finally, choose whichever mapped_num_proc is lower
+ )
+
+ num_shards = min(existing_cache_file_map, key=select_existing_cache_files)
+
+ def format_cache_file_name(
+ cache_file_name: Optional[str],
+ rank: Union[int, Literal["*"]], # noqa: F722
+ ) -> Optional[str]:
+ if not cache_file_name:
+ return cache_file_name
+
+ assert cache_file_prefix is not None and cache_file_ext is not None
+
+ if isinstance(rank, int):
+ cache_file_name = (
+ cache_file_prefix + suffix_template.format(rank=rank, num_proc=num_shards) + cache_file_ext
+ )
+ if not os.path.exists(cache_file_name):
+ process_name = (
+ "Main process" if num_proc is None or num_proc == 1 else f"Process #{rank % num_shards + 1}"
+ )
+ logger.info(f"{process_name} will write at {cache_file_name}")
+ else:
+ # TODO: this assumes the format_spec of rank in suffix_template
+ cache_file_name = (
+ cache_file_prefix
+ + suffix_template.replace("{rank:05d}", "{rank}").format(rank=rank, num_proc=num_shards)
+ + cache_file_ext
+ )
+ return cache_file_name
+
+ def format_new_fingerprint(new_fingerprint: str, rank: int) -> str:
+ new_fingerprint = new_fingerprint + suffix_template.format(rank=rank, num_proc=num_shards)
+ validate_fingerprint(new_fingerprint)
+ return new_fingerprint
+
+ if num_proc is not None and num_proc >= 1:
+ prev_env = deepcopy(os.environ)
+ # check if parallelism if off
+ # from https://github.com/huggingface/tokenizers/blob/bb668bc439dc34389b71dbb8ce0c597f15707b53/tokenizers/src/utils/parallelism.rs#L22
+ if prev_env.get("TOKENIZERS_PARALLELISM", "false").lower() not in (
+ "",
+ "off",
+ "false",
+ "f",
+ "no",
+ "n",
+ "0",
+ ):
+ logger.warning("Setting TOKENIZERS_PARALLELISM=false for forked processes.")
+ os.environ["TOKENIZERS_PARALLELISM"] = "false"
+ else:
+ prev_env = os.environ
+
+ kwargs_per_job: list[Optional[dict[str, Any]]]
+ if num_shards == 1:
+ shards = [self]
+ kwargs_per_job = [dataset_kwargs]
+ else:
+ shards = [
+ self.shard(num_shards=num_shards, index=rank, contiguous=True, keep_in_memory=keep_in_memory)
+ for rank in range(num_shards)
+ ]
+ kwargs_per_job = [
+ {
+ **dataset_kwargs,
+ "shard": shards[rank],
+ "cache_file_name": format_cache_file_name(cache_file_name, rank),
+ "rank": rank,
+ "offset": sum(len(s) for s in shards[:rank]),
+ "new_fingerprint": format_new_fingerprint(new_fingerprint, rank),
+ }
+ for rank in range(num_shards)
+ ]
+
+ transformed_shards: list[Optional[Dataset]] = [None] * num_shards
+ for rank in range(num_shards):
+ try:
+ job_kwargs = kwargs_per_job[rank]
+ assert job_kwargs is not None
+ transformed_shards[rank] = load_processed_shard_from_cache(job_kwargs)
+ kwargs_per_job[rank] = None
+ except NonExistentDatasetError:
+ pass
+
+ if unprocessed_kwargs_per_job := [kwargs for kwargs in kwargs_per_job if kwargs is not None]:
+ if len(unprocessed_kwargs_per_job) != num_shards:
+ logger.info(
+ f"Reprocessing {len(unprocessed_kwargs_per_job)}/{num_shards} shards because some of them were "
+ "missing from the cache."
+ )
+
+ pbar_total = len(self)
+ num_shards_done = num_shards - len(unprocessed_kwargs_per_job)
+ pbar_initial = num_shards_done * pbar_total // num_shards
+ if batched and drop_last_batch:
+ batch_size = batch_size or 1
+ pbar_initial = pbar_initial // num_shards // batch_size * num_shards * batch_size
+ pbar_total = pbar_total // num_shards // batch_size * num_shards * batch_size
+
+ with hf_tqdm(
+ unit=" examples",
+ initial=pbar_initial,
+ total=pbar_total,
+ desc=(desc or "Map") + (f" (num_proc={num_proc})" if num_proc is not None and num_proc >= 1 else ""),
+ ) as pbar:
+ shards_done = 0
+
+ def check_if_shard_done(rank: Optional[int], done: bool, content: Union[Dataset, int]) -> None:
+ nonlocal shards_done
+ if done:
+ shards_done += 1
+ logger.debug(f"Finished processing shard number {rank} of {num_shards}.")
+ assert isinstance(content, Dataset)
+ transformed_shards[rank or 0] = content
+ else:
+ assert isinstance(content, int)
+ pbar.update(content)
+
+ if num_proc is not None and num_proc >= 1:
+ with mp.Pool(num_proc) as pool:
+ os.environ = prev_env
+ logger.info(f"Spawning {num_proc} processes")
+
+ for rank, done, content in iflatmap_unordered(
+ pool, Dataset._map_single, kwargs_iterable=unprocessed_kwargs_per_job
+ ):
+ check_if_shard_done(rank, done, content)
+
+ pool.close()
+ pool.join()
+ else:
+ for unprocessed_kwargs in unprocessed_kwargs_per_job:
+ for rank, done, content in Dataset._map_single(**unprocessed_kwargs):
+ check_if_shard_done(rank, done, content)
+
+ # Avoids PermissionError on Windows (the error: https://github.com/huggingface/datasets/actions/runs/4026734820/jobs/6921621805)
+ for job_kwargs in unprocessed_kwargs_per_job:
+ if "shard" in job_kwargs:
+ del job_kwargs["shard"]
+ else:
+ logger.info(f"Loading cached processed dataset at {format_cache_file_name(cache_file_name, '*')}")
+
+ all_transformed_shards = [shard for shard in transformed_shards if shard is not None]
+ if len(transformed_shards) != len(all_transformed_shards):
+ raise ValueError(
+ f"Failed to retrieve results from map: result list {transformed_shards} still contains None - "
+ "at least one worker failed to return its results"
+ )
+
+ if num_shards == 1:
+ result = all_transformed_shards[0]
+ else:
+ logger.info(f"Concatenating {num_shards} shards")
+ result = _concatenate_map_style_datasets(all_transformed_shards)
+
+ # update fingerprint if the dataset changed
+ result._fingerprint = (
+ new_fingerprint
+ if any(
+ transformed_shard._fingerprint != shard._fingerprint
+ for transformed_shard, shard in zip(all_transformed_shards, shards)
+ )
+ else self._fingerprint
+ )
+
+ return result
+
+ @staticmethod
+ def _map_single(
+ shard: "Dataset",
+ function: Optional[Callable] = None,
+ with_indices: bool = False,
+ with_rank: bool = False,
+ input_columns: Optional[list[str]] = None,
+ batched: bool = False,
+ batch_size: Optional[int] = 1000,
+ drop_last_batch: bool = False,
+ remove_columns: Optional[list[str]] = None,
+ keep_in_memory: bool = False,
+ cache_file_name: Optional[str] = None,
+ writer_batch_size: Optional[int] = 1000,
+ features: Optional[Features] = None,
+ disable_nullable: bool = False,
+ fn_kwargs: Optional[dict] = None,
+ new_fingerprint: Optional[str] = None,
+ rank: Optional[int] = None,
+ offset: int = 0,
+ try_original_type: Optional[bool] = True,
+ on_mixed_types: Optional[Literal["use_json"]] = "use_json",
+ ) -> Iterable[tuple[Optional[int], bool, Union[int, "Dataset"]]]:
+ """Apply a function to all the elements in the table (individually or in batches)
+ and update the table (if function does update examples).
+
+ Args:
+ shard (`datasets.Dataset`): Dataset to map the transform on.
+ function (`Callable`): with one of the following signature:
+ - `function(example: Dict[str, Any]) -> Dict[str, Any]` if `batched=False` and `with_indices=False` and `with_rank=False`
+ - `function(example: Dict[str, Any], *extra_args) -> Dict[str, Any]` if `batched=False` and `with_indices=True` and/or `with_rank=True` (one extra arg for each)
+ - `function(batch: Dict[str, List]) -> Dict[str, List]` if `batched=True` and `with_indices=False` and `with_rank=False`
+ - `function(batch: Dict[str, List], *extra_args) -> Dict[str, List]` if `batched=True` and `with_indices=True` and/or `with_rank=True` (one extra arg for each)
+
+ For advanced usage, the function can also return a `pyarrow.Table`.
+ Moreover if your function returns nothing (`None`), then `map` will run your function and return the dataset unchanged.
+ If no function is provided, default to identity function: lambda x: x
+ with_indices (`bool`, defaults to `False`): Provide example indices to `function`. Note that in this case the signature of `function` should be `def function(example, idx[, rank]): ...`.
+ with_rank (`bool`, default `False`): Provide process rank to `function`. Note that in this case the signature of `function` should be `def function(example[, idx], rank): ...`.
+ input_columns (`Optional[List[str]]`, defaults to `None`): The columns to be passed into `function` as
+ positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument.
+ batched (`bool`, defaults to `False`): Provide batch of examples to `function`
+ batch_size (`int`, optional, defaults to `1000`): Number of examples per batch provided to `function` if `batched=True`
+ `batch_size <= 0` or `batch_size == None`: Provide the full dataset as a single batch to `function`
+ drop_last_batch (`bool`, default: `False`): Whether a last batch smaller than the batch_size should be
+ dropped instead of being processed by the function.
+ remove_columns (`Optional[List[str]]`, defaults to `None`): Remove a selection of columns while doing the mapping.
+ Columns will be removed before updating the examples with the output of `function`, i.e. if `function` is adding
+ columns with names in `remove_columns`, these columns will be kept.
+ keep_in_memory (`bool`, defaults to `False`): Keep the dataset in memory instead of writing it to a cache file.
+ cache_file_name (`str`, optional, defaults to `None`): Provide the name of a path for the cache file. It is used to store the
+ results of the computation instead of the automatically generated cache file name.
+ writer_batch_size (`int`, default `1000`): Number of rows per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `.map()`.
+ features (`Optional[datasets.Features]`, defaults to `None`): Use a specific Features to store the cache file
+ instead of the automatically generated one.
+ disable_nullable (`bool`, defaults to `False`): Disallow null values in the table.
+ fn_kwargs (`Dict`, optional, defaults to `None`): Keyword arguments to be passed to `function`
+ new_fingerprint (`str`, optional, defaults to `None`): the new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments
+ rank: (`int`, optional, defaults to `None`): If specified, this is the process rank when doing multiprocessing
+ offset: (`int`, defaults to 0): If specified, this is an offset applied to the indices passed to `function` if `with_indices=True`.
+ try_original_type: (`Optional[bool]`, defaults to `True`):
+ Try to keep the types of the original columns (e.g. int32 -> int32).
+ Set to False if you want to always infer new types.
+ on_mixed_types (`Literal["use_json"]`, *optional*, defaults to `None`):
+ If "use_json", use the Json() type for mixed-types fields,
+ i.e. unstructured fields that contain data without a predefined schema.
+ In this case, a field with mixed type is set to Json().
+
+ This allow loading lists with a mix of strings/integers/floats
+ for example, or dictionaries with arbitrary value types.
+
+
+ """
+ if fn_kwargs is None:
+ fn_kwargs = {}
+
+ # If we do batch computation but no batch size is provided, default to the full dataset
+ if batched and (batch_size is None or batch_size <= 0):
+ batch_size = shard.num_rows
+
+ # We set this variable to True after processing the first example/batch in
+ # `apply_function_on_filtered_inputs` if the map function returns a dict.
+ # If set to False, no new arrow table will be created
+
+ update_data = None
+
+ format_kwargs = shard._format_kwargs.copy()
+ # Lazy formatting is only available for the default format (None/python)
+ if not input_columns and shard._format_type is None:
+ format_kwargs["lazy"] = True
+ input_formatter = get_formatter(
+ shard._format_type,
+ features=shard.features,
+ **format_kwargs,
+ )
+
+ check_same_num_examples = batched and len(shard.list_indexes()) > 0
+
+ def validate_function_output(processed_inputs):
+ """Validate output of the map function."""
+ allowed_processed_inputs_types = (Mapping, pa.Table, pd.DataFrame)
+ if config.POLARS_AVAILABLE and "polars" in sys.modules:
+ import polars as pl
+
+ allowed_processed_inputs_types += (pl.DataFrame,)
+ if processed_inputs is not None and not isinstance(processed_inputs, allowed_processed_inputs_types):
+ raise TypeError(
+ f"Provided `function` which is applied to all elements of table returns a variable of type {type(processed_inputs)}. Make sure provided `function` returns a variable of type `dict` (or a pyarrow table) to update the dataset or `None` if you are only interested in side effects."
+ )
+ if batched and isinstance(processed_inputs, Mapping):
+ allowed_batch_return_types = (list, np.ndarray, pd.Series)
+ if config.POLARS_AVAILABLE and "polars" in sys.modules:
+ import polars as pl
+
+ allowed_batch_return_types += (pl.Series, pl.DataFrame)
+ if config.TF_AVAILABLE and "tensorflow" in sys.modules:
+ import tensorflow as tf
+
+ allowed_batch_return_types += (tf.Tensor,)
+ if config.TORCH_AVAILABLE and "torch" in sys.modules:
+ import torch
+
+ allowed_batch_return_types += (torch.Tensor,)
+ if config.JAX_AVAILABLE and "jax" in sys.modules:
+ import jax.numpy as jnp
+
+ allowed_batch_return_types += (jnp.ndarray,)
+ all_dict_values_are_lists = all(
+ isinstance(value, allowed_batch_return_types) for value in processed_inputs.values()
+ )
+ if all_dict_values_are_lists is False:
+ raise TypeError(
+ f"Provided `function` which is applied to all elements of table returns a `dict` of types {[type(x) for x in processed_inputs.values()]}. When using `batched=True`, make sure provided `function` returns a `dict` of types like `{allowed_batch_return_types}`."
+ )
+
+ def prepare_inputs(pa_inputs, indices, offset=0):
+ """Utility to apply the function on a selection of columns."""
+ inputs = format_table(
+ pa_inputs,
+ 0 if not batched else range(pa_inputs.num_rows),
+ format_columns=input_columns,
+ formatter=input_formatter,
+ )
+ fn_args = [inputs] if input_columns is None else [inputs[col] for col in input_columns]
+ if offset == 0:
+ effective_indices = indices
+ else:
+ effective_indices = [i + offset for i in indices] if isinstance(indices, list) else indices + offset
+ additional_args = ()
+ if with_indices:
+ additional_args += (effective_indices,)
+ if with_rank:
+ additional_args += (rank,)
+ return inputs, fn_args, additional_args, fn_kwargs
+
+ def prepare_outputs(pa_inputs, inputs, processed_inputs):
+ nonlocal update_data
+ if not (update_data := (processed_inputs is not None)):
+ return None
+ if isinstance(processed_inputs, LazyDict):
+ processed_inputs = {
+ k: v for k, v in processed_inputs.data.items() if k not in processed_inputs.keys_to_format
+ }
+ returned_lazy_dict = True
+ else:
+ returned_lazy_dict = False
+ validate_function_output(processed_inputs)
+ if shard._format_type or input_columns:
+ # TODO(QL, MS): ideally the behavior should be the same even if the dataset is formatted (may require major release)
+ inputs_to_merge = dict(zip(pa_inputs.column_names, pa_inputs.itercolumns()))
+ elif isinstance(inputs, LazyDict):
+ inputs_to_merge = {
+ k: (v if k not in inputs.keys_to_format else pa_inputs[k]) for k, v in inputs.data.items()
+ }
+ else:
+ inputs_to_merge = inputs
+ if remove_columns is not None:
+ for column in remove_columns:
+ # `function` can modify input in-place causing column to be already removed.
+ if column in inputs_to_merge:
+ inputs_to_merge.pop(column)
+ if returned_lazy_dict and column in processed_inputs:
+ processed_inputs.pop(column)
+ if check_same_num_examples:
+ input_num_examples = len(pa_inputs)
+ processed_inputs_num_examples = len(processed_inputs[next(iter(processed_inputs.keys()))])
+ if input_num_examples != processed_inputs_num_examples:
+ raise DatasetTransformationNotAllowedError(
+ "Using `.map` in batched mode on a dataset with attached indexes is allowed only if it doesn't create or remove existing examples. You can first run `.drop_index() to remove your index and then re-add it."
+ ) from None
+ if isinstance(inputs, Mapping) and isinstance(processed_inputs, Mapping):
+ # The .map() transform *updates* the dataset:
+ # the output dictionary contains both the input data and the output data.
+ # The output dictionary may contain Arrow values from `inputs_to_merge` so that we can re-write them efficiently.
+ return {**inputs_to_merge, **processed_inputs}
+ else:
+ return processed_inputs
+
+ def apply_function(pa_inputs, indices, offset=0):
+ """Utility to apply the function on a selection of columns."""
+ inputs, fn_args, additional_args, fn_kwargs = prepare_inputs(pa_inputs, indices, offset=offset)
+ processed_inputs = function(*fn_args, *additional_args, **fn_kwargs)
+ return prepare_outputs(pa_inputs, inputs, processed_inputs)
+
+ async def async_apply_function(pa_inputs, indices, offset=0):
+ """Utility to apply the function on a selection of columns. Same code but async"""
+ inputs, fn_args, additional_args, fn_kwargs = prepare_inputs(pa_inputs, indices, offset=offset)
+ processed_inputs = await function(*fn_args, *additional_args, **fn_kwargs)
+ return prepare_outputs(pa_inputs, inputs, processed_inputs)
+
+ def init_buffer_and_writer():
+ # Prepare output buffer and batched writer in memory or on file if we update the table
+ writer_features = features
+ if writer_features is None:
+ writer_features = shard.features
+ update_features = True
+ else:
+ update_features = False
+ if keep_in_memory or cache_file_name is None:
+ buf_writer = pa.BufferOutputStream()
+ tmp_file = None
+ writer = ArrowWriter(
+ features=writer_features,
+ stream=buf_writer,
+ writer_batch_size=writer_batch_size,
+ update_features=update_features,
+ fingerprint=new_fingerprint,
+ disable_nullable=disable_nullable,
+ on_mixed_types=on_mixed_types,
+ )
+ else:
+ buf_writer = None
+ logger.info(f"Caching processed dataset at {cache_file_name}")
+ cache_dir = os.path.dirname(cache_file_name)
+ os.makedirs(cache_dir, exist_ok=True)
+ tmp_file = tempfile.NamedTemporaryFile("wb", dir=cache_dir, delete=False)
+ writer = ArrowWriter(
+ features=writer_features,
+ path=tmp_file.name,
+ writer_batch_size=writer_batch_size,
+ update_features=update_features,
+ fingerprint=new_fingerprint,
+ disable_nullable=disable_nullable,
+ on_mixed_types=on_mixed_types,
+ )
+ return buf_writer, writer, tmp_file
+
+ tasks: list[asyncio.Task] = []
+ if inspect.iscoroutinefunction(function):
+ try:
+ loop = asyncio.get_running_loop()
+ except RuntimeError:
+ loop = asyncio.new_event_loop()
+ else:
+ loop = None
+
+ def iter_outputs(shard_iterable):
+ nonlocal tasks, loop
+ if inspect.iscoroutinefunction(function):
+ indices: Union[list[int], list[list[int]]] = []
+ for i, example in shard_iterable:
+ indices.append(i)
+ tasks.append(loop.create_task(async_apply_function(example, i, offset=offset)))
+ # keep the total active tasks under a certain number
+ if len(tasks) >= config.MAX_NUM_RUNNING_ASYNC_MAP_FUNCTIONS_IN_PARALLEL:
+ done, pending = loop.run_until_complete(
+ asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+ )
+ while tasks and len(pending) >= config.MAX_NUM_RUNNING_ASYNC_MAP_FUNCTIONS_IN_PARALLEL:
+ done, pending = loop.run_until_complete(
+ asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+ )
+ # yield finished tasks
+ while tasks and tasks[0].done():
+ yield indices.pop(0), tasks.pop(0).result()
+ while tasks:
+ yield indices[0], loop.run_until_complete(tasks[0])
+ indices.pop(0), tasks.pop(0)
+ else:
+ for i, example in shard_iterable:
+ yield i, apply_function(example, i, offset=offset)
+
+ num_examples_progress_update = 0
+ # If `update_data` is True after processing the first example/batch, initialize these resources with `init_buffer_and_writer`
+ buf_writer, writer, tmp_file = None, None, None
+
+ # Optionally initialize the writer as a context manager
+ with contextlib.ExitStack() as stack:
+ try:
+ arrow_formatted_shard = shard.with_format("arrow")
+
+ # Loop over single examples or batches and write to buffer/file if examples are to be updated
+ if not batched:
+ shard_iterable = enumerate(arrow_formatted_shard)
+ else:
+ num_rows = len(shard) if not drop_last_batch else len(shard) // batch_size * batch_size
+ shard_iterable = zip(
+ (list(range(i, min(i + batch_size, num_rows))) for i in range(0, num_rows, batch_size)),
+ arrow_formatted_shard.iter(batch_size, drop_last_batch=drop_last_batch),
+ )
+ if not batched:
+ _time = time.time()
+ for i, example in iter_outputs(shard_iterable):
+ if update_data:
+ if i == 0:
+ buf_writer, writer, tmp_file = init_buffer_and_writer()
+ stack.enter_context(writer)
+ if isinstance(example, pa.Table):
+ writer.write_row(example)
+ elif isinstance(example, pd.DataFrame):
+ writer.write_row(pa.Table.from_pandas(example))
+ elif config.POLARS_AVAILABLE and "polars" in sys.modules:
+ import polars as pl
+
+ if isinstance(example, pl.DataFrame):
+ writer.write_row(example.to_arrow())
+ else:
+ writer.write(example)
+ else:
+ writer.write(example)
+ num_examples_progress_update += 1
+ if time.time() > _time + config.PBAR_REFRESH_TIME_INTERVAL:
+ _time = time.time()
+ yield rank, False, num_examples_progress_update
+ num_examples_progress_update = 0
+ else:
+ _time = time.time()
+ for i, batch in iter_outputs(shard_iterable):
+ num_examples_in_batch = len(i)
+ if update_data:
+ if i and i[0] == 0:
+ buf_writer, writer, tmp_file = init_buffer_and_writer()
+ stack.enter_context(writer)
+ if isinstance(batch, pa.Table):
+ writer.write_table(batch)
+ elif isinstance(batch, pd.DataFrame):
+ writer.write_table(pa.Table.from_pandas(batch))
+ elif config.POLARS_AVAILABLE and "polars" in sys.modules:
+ import polars as pl
+
+ if isinstance(batch, pl.DataFrame):
+ writer.write_table(batch.to_arrow())
+ else:
+ writer.write_batch(batch, try_original_type=try_original_type)
+ else:
+ writer.write_batch(batch, try_original_type=try_original_type)
+ num_examples_progress_update += num_examples_in_batch
+ if time.time() > _time + config.PBAR_REFRESH_TIME_INTERVAL:
+ _time = time.time()
+ yield rank, False, num_examples_progress_update
+ num_examples_progress_update = 0
+ if update_data and writer is not None:
+ writer.finalize() # close_stream=bool(buf_writer is None)) # We only close if we are writing in a file
+ except (Exception, KeyboardInterrupt):
+ yield rank, False, num_examples_progress_update
+ if update_data:
+ if writer is not None:
+ writer.finalize()
+ if tmp_file is not None:
+ tmp_file.close()
+ if os.path.exists(tmp_file.name):
+ os.remove(tmp_file.name)
+ if loop:
+ logger.debug(f"Canceling {len(tasks)} async tasks.")
+ for task in tasks:
+ task.cancel(msg="KeyboardInterrupt")
+ try:
+ loop.run_until_complete(asyncio.gather(*tasks))
+ except (asyncio.CancelledError, ValueError):
+ logger.debug("Tasks canceled.")
+ raise
+
+ yield rank, False, num_examples_progress_update
+ if update_data and tmp_file is not None:
+ tmp_file.close()
+ shutil.move(tmp_file.name, cache_file_name)
+ umask = os.umask(0o666)
+ os.umask(umask)
+ os.chmod(cache_file_name, 0o666 & ~umask)
+
+ if update_data:
+ # Create new Dataset from buffer or file
+ info = shard.info.copy()
+ info.features = writer._features
+ if buf_writer is None:
+ yield rank, True, Dataset.from_file(cache_file_name, info=info, split=shard.split)
+ else:
+ yield rank, True, Dataset.from_buffer(buf_writer.getvalue(), info=info, split=shard.split)
+ else:
+ yield rank, True, shard
+
+ @transmit_format
+ @fingerprint_transform(inplace=False)
+ def batch(
+ self,
+ batch_size: Optional[int] = None,
+ by_column: Optional[Union[str, list[str]]] = None,
+ drop_last_batch: bool = False,
+ num_proc: Optional[int] = None,
+ new_fingerprint: Optional[str] = None,
+ ) -> "Dataset":
+ """
+ Group samples from the dataset into batches.
+
+ Specify either `batch_size` to make batches of `batch_size` examples.
+
+ Or use `by_column=column_name` to make batches of successive examples that share
+ the same value in column `column_name`.
+
+ Args:
+ batch_size (`int`, optional):
+ The number of samples in each batch.
+ by_column (`Union[str, list[str]`, optional):
+ The column used to batch examples together.
+ Successive examples with the same value for that column are in grouped the same batch.
+ This can also be a list of columns if you want to batch by multiple columns.
+ If batching by column, the batch_size is only used to control the size of the batches
+ to group together or slice during acculumation.
+
+
+ drop_last_batch (`bool`, defaults to `False`):
+ Whether to drop the last incomplete batch.
+ num_proc (`int`, *optional*, defaults to `None`):
+ Max number of processes when generating cache. Already cached shards are loaded sequentially.
+ new_fingerprint (`str`, *optional*, defaults to `None`):
+ The new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments.
+
+ Returns:
+ [`Dataset`]: A new Dataset where each item is a batch of multiple samples from the original dataset.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
+ >>> batched_ds = ds.batch(batch_size=4)
+ >>> batched_ds[0]
+ {'text': ['compassionately explores the seemingly irreconcilable situation...', ...], # 4 items
+ 'label': [1, 1, 1, 1]}
+ ```
+ """
+ if batch_size is None and by_column is None:
+ raise ValueError("IterableDataset.batch() misses `batch_size` or `by_column` argument.")
+ if by_column is not None:
+ if num_proc:
+ raise NotImplementedError("Multiprocessed batching by column is not implemented yet")
+ columns = [by_column] if isinstance(by_column, str) else by_column
+ _batch_fn = partial(
+ _batch_accumulate_arrow_table_by_columns, columns=columns, tables_accumulator=[], length=len(self)
+ )
+ with_indices = True
+ else:
+ _batch_fn = _batch_arrow_table
+ with_indices = False
+
+ return self.with_format("arrow").map(
+ _batch_fn,
+ with_indices=with_indices,
+ batched=True,
+ batch_size=batch_size,
+ drop_last_batch=drop_last_batch,
+ num_proc=num_proc,
+ new_fingerprint=new_fingerprint,
+ desc="Batching examples",
+ features=Features({col: List(feature) for col, feature in self.features.items()}),
+ )
+
+ @transmit_format
+ @fingerprint_transform(
+ inplace=False, ignore_kwargs=["load_from_cache_file", "cache_file_name", "desc"], version="2.0.1"
+ )
+ def filter(
+ self,
+ function: Optional[Callable] = None,
+ with_indices: bool = False,
+ with_rank: bool = False,
+ input_columns: Optional[Union[str, list[str]]] = None,
+ batched: bool = False,
+ batch_size: Optional[int] = 1000,
+ keep_in_memory: bool = False,
+ load_from_cache_file: Optional[bool] = None,
+ cache_file_name: Optional[str] = None,
+ writer_batch_size: Optional[int] = 1000,
+ fn_kwargs: Optional[dict] = None,
+ num_proc: Optional[int] = None,
+ suffix_template: str = "_{rank:05d}_of_{num_proc:05d}",
+ new_fingerprint: Optional[str] = None,
+ desc: Optional[str] = None,
+ ) -> "Dataset":
+ """Apply a filter function to all the elements in the table in batches
+ and update the table so that the dataset only includes examples according to the filter function.
+
+ If the function is asynchronous, then `filter` will run your function in parallel, with up to one thousand simultaneous calls (configurable).
+ It is recommended to use a `asyncio.Semaphore` in your function if you want to set a maximum number of operations that can run at the same time.
+
+ Args:
+ function (`Callable`): Callable with one of the following signatures:
+
+ - `function(example: Dict[str, Any]) -> bool` if `batched=False` and `with_indices=False` and `with_rank=False`
+ - `function(example: Dict[str, Any], *extra_args) -> bool` if `batched=False` and `with_indices=True` and/or `with_rank=True` (one extra arg for each)
+ - `function(batch: Dict[str, List]) -> List[bool]` if `batched=True` and `with_indices=False` and `with_rank=False`
+ - `function(batch: Dict[str, List], *extra_args) -> List[bool]` if `batched=True` and `with_indices=True` and/or `with_rank=True` (one extra arg for each)
+
+ If the function is asynchronous, then `filter` will run your function in parallel.
+ If no function is provided, defaults to an always `True` function: `lambda x: True`.
+ with_indices (`bool`, defaults to `False`):
+ Provide example indices to `function`. Note that in this case the
+ signature of `function` should be `def function(example, idx[, rank]): ...`.
+ with_rank (`bool`, defaults to `False`):
+ Provide process rank to `function`. Note that in this case the
+ signature of `function` should be `def function(example[, idx], rank): ...`.
+ input_columns (`str` or `List[str]`, *optional*):
+ The columns to be passed into `function` as
+ positional arguments. If `None`, a `dict` mapping to all formatted columns is passed as one argument.
+ batched (`bool`, defaults to `False`):
+ Provide batch of examples to `function`.
+ batch_size (`int`, *optional*, defaults to `1000`):
+ Number of examples per batch provided to `function` if
+ `batched = True`. If `batched = False`, one example per batch is passed to `function`.
+ If `batch_size <= 0` or `batch_size == None`, provide the full dataset as a single batch to `function`.
+ keep_in_memory (`bool`, defaults to `False`):
+ Keep the dataset in memory instead of writing it to a cache file.
+ load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled):
+ If a cache file storing the current computation from `function`
+ can be identified, use it instead of recomputing.
+ cache_file_name (`str`, *optional*):
+ Provide the name of a path for the cache file. It is used to store the
+ results of the computation instead of the automatically generated cache file name.
+ writer_batch_size (`int`, defaults to `1000`):
+ Number of rows per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`.
+ fn_kwargs (`dict`, *optional*):
+ Keyword arguments to be passed to `function`.
+ num_proc (`int`, *optional*, defaults to `None`):
+ The number of processes to use for multiprocessing.
+ - If `None` or `0`, no multiprocessing is used and the operation runs in the main process.
+ - If greater than `1`, one or multiple worker processes are used to process data in parallel.
+ Note: The function passed to `map()` must be picklable for multiprocessing to work correctly
+ (i.e., prefer functions defined at the top level of a module, not inside another function or class).
+ suffix_template (`str`):
+ If `cache_file_name` is specified, then this suffix will be added at the end of the base name of each.
+ For example, if `cache_file_name` is `"processed.arrow"`, then for `rank = 1` and `num_proc = 4`,
+ the resulting file would be `"processed_00001_of_00004.arrow"` for the default suffix (default
+ `_{rank:05d}_of_{num_proc:05d}`).
+ new_fingerprint (`str`, *optional*):
+ The new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments.
+ desc (`str`, *optional*, defaults to `None`):
+ Meaningful description to be displayed alongside with the progress bar while filtering examples.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds = ds.filter(lambda x: x["label"] == 1)
+ >>> ds
+ Dataset({
+ features: ['text', 'label'],
+ num_rows: 533
+ })
+ ```
+
+ """
+ if len(self.list_indexes()) > 0:
+ raise DatasetTransformationNotAllowedError(
+ "Using `.filter` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it.`"
+ )
+
+ if function is None:
+ function = lambda x: True # noqa: E731
+
+ if len(self) == 0:
+ return self
+
+ # We generally batch the underlying map() to get faster throughput,
+ # but in case of async we force batch_size=1 to enable parallelism
+ if inspect.iscoroutinefunction(function) and not batched:
+ batch_size = 1
+
+ indices = self.map(
+ function=partial(
+ async_get_indices_from_mask_function
+ if inspect.iscoroutinefunction(function)
+ else get_indices_from_mask_function,
+ function,
+ batched,
+ with_indices,
+ with_rank,
+ input_columns,
+ self._indices,
+ ),
+ with_indices=True,
+ with_rank=True,
+ features=Features({"indices": Value("uint64")}),
+ batched=True,
+ batch_size=batch_size,
+ remove_columns=self.column_names,
+ keep_in_memory=keep_in_memory,
+ load_from_cache_file=load_from_cache_file,
+ cache_file_name=cache_file_name,
+ writer_batch_size=writer_batch_size,
+ fn_kwargs=fn_kwargs,
+ num_proc=num_proc,
+ suffix_template=suffix_template,
+ new_fingerprint=new_fingerprint,
+ input_columns=input_columns,
+ desc=desc or "Filter",
+ )
+ new_dataset = copy.deepcopy(self)
+ new_dataset._indices = indices.data
+ new_dataset._fingerprint = new_fingerprint
+ return new_dataset
+
+ @transmit_format
+ @fingerprint_transform(inplace=False, ignore_kwargs=["cache_file_name"])
+ def flatten_indices(
+ self,
+ keep_in_memory: bool = False,
+ cache_file_name: Optional[str] = None,
+ writer_batch_size: Optional[int] = 1000,
+ features: Optional[Features] = None,
+ disable_nullable: bool = False,
+ num_proc: Optional[int] = None,
+ new_fingerprint: Optional[str] = None,
+ ) -> "Dataset":
+ """Create and cache a new Dataset by flattening the indices mapping.
+
+ Args:
+ keep_in_memory (`bool`, defaults to `False`):
+ Keep the dataset in memory instead of writing it to a cache file.
+ cache_file_name (`str`, *optional*, default `None`):
+ Provide the name of a path for the cache file. It is used to store the
+ results of the computation instead of the automatically generated cache file name.
+ writer_batch_size (`int`, defaults to `1000`):
+ Number of rows per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`.
+ features (`Optional[datasets.Features]`, defaults to `None`):
+ Use a specific [`Features`] to store the cache file
+ instead of the automatically generated one.
+ disable_nullable (`bool`, defaults to `False`):
+ Allow null values in the table.
+ num_proc (`int`, optional, default `None`):
+ Max number of processes when generating cache. Already cached shards are loaded sequentially
+ new_fingerprint (`str`, *optional*, defaults to `None`):
+ The new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments
+ """
+
+ return self.map(
+ batched=True, # for speed
+ keep_in_memory=keep_in_memory,
+ cache_file_name=cache_file_name,
+ writer_batch_size=writer_batch_size,
+ features=features,
+ disable_nullable=disable_nullable,
+ new_fingerprint=new_fingerprint,
+ desc="Flattening the indices",
+ num_proc=num_proc,
+ )
+
+ def _new_dataset_with_indices(
+ self,
+ indices_cache_file_name: Optional[str] = None,
+ indices_buffer: Optional[pa.Buffer] = None,
+ fingerprint: Optional[str] = None,
+ ) -> "Dataset":
+ """Return a new Dataset obtained by adding indices (provided in indices_cache_file_name or in a buffer) to the
+ current Dataset.
+ """
+
+ if indices_cache_file_name is None and indices_buffer is None:
+ raise ValueError("At least one of indices_cache_file_name or indices_buffer must be provided.")
+
+ if fingerprint is None:
+ raise ValueError("please specify a fingerprint for the dataset with indices")
+
+ if indices_cache_file_name is not None:
+ indices_table = MemoryMappedTable.from_file(indices_cache_file_name)
+ else:
+ indices_table = InMemoryTable.from_buffer(indices_buffer)
+
+ # Return new Dataset object
+ # don't forget to copy the objects
+ return Dataset(
+ self._data,
+ info=self.info.copy(),
+ split=self.split,
+ indices_table=indices_table,
+ fingerprint=fingerprint,
+ )
+
+ @transmit_format
+ @fingerprint_transform(inplace=False, ignore_kwargs=["indices_cache_file_name"])
+ def select(
+ self,
+ indices: Iterable,
+ keep_in_memory: bool = False,
+ indices_cache_file_name: Optional[str] = None,
+ writer_batch_size: Optional[int] = 1000,
+ new_fingerprint: Optional[str] = None,
+ ) -> "Dataset":
+ """Create a new dataset with rows selected following the list/array of indices.
+
+ Args:
+ indices (`range`, `list`, `iterable`, `ndarray` or `Series`):
+ Range, list or 1D-array of integer indices for indexing.
+ If the indices correspond to a contiguous range, the Arrow table is simply sliced.
+ However passing a list of indices that are not contiguous creates indices mapping, which is much less efficient,
+ but still faster than recreating an Arrow table made of the requested rows.
+ keep_in_memory (`bool`, defaults to `False`):
+ Keep the indices mapping in memory instead of writing it to a cache file.
+ indices_cache_file_name (`str`, *optional*, defaults to `None`):
+ Provide the name of a path for the cache file. It is used to store the
+ indices mapping instead of the automatically generated cache file name.
+ writer_batch_size (`int`, defaults to `1000`):
+ Number of rows per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`.
+ new_fingerprint (`str`, *optional*, defaults to `None`):
+ The new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds = ds.select(range(4))
+ >>> ds
+ Dataset({
+ features: ['text', 'label'],
+ num_rows: 4
+ })
+ ```
+ """
+ if keep_in_memory and indices_cache_file_name is not None:
+ raise ValueError("Please use either `keep_in_memory` or `indices_cache_file_name` but not both.")
+
+ if len(self.list_indexes()) > 0:
+ raise DatasetTransformationNotAllowedError(
+ "Using `.select` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it."
+ )
+
+ # If the array is empty we do nothing
+ if len(self) == 0:
+ return self
+
+ # If indices is a PyArrow array, we convert to NumPy
+ if isinstance(indices, (pa.Array, pa.ChunkedArray)):
+ indices = indices.to_numpy().astype(np.int64)
+
+ # Convert generator objects to lists
+ if isinstance(indices, Iterator):
+ indices = list(indices)
+
+ # If the indices are contiguous, simply slice the arrow table
+ if isinstance(indices, range):
+ if _is_range_contiguous(indices) and indices.start >= 0:
+ start, length = indices.start, indices.stop - indices.start
+ return self._select_contiguous(start, length, new_fingerprint=new_fingerprint)
+ else:
+ try:
+ start = next(iter(indices))
+ except StopIteration:
+ # if `indices` is an empty iterable, we return an empty dataset
+ return self._select_contiguous(0, 0, new_fingerprint=new_fingerprint)
+ if start >= 0:
+ counter_from_start = itertools.count(start=start)
+ if all(i == j for i, j in zip(indices, counter_from_start)):
+ length = next(counter_from_start) - start
+ return self._select_contiguous(start, length, new_fingerprint=new_fingerprint)
+
+ # If not contiguous, we need to create a new indices mapping
+ return self._select_with_indices_mapping(
+ indices,
+ keep_in_memory=keep_in_memory,
+ indices_cache_file_name=indices_cache_file_name,
+ writer_batch_size=writer_batch_size,
+ new_fingerprint=new_fingerprint,
+ )
+
+ @transmit_format
+ @fingerprint_transform(inplace=False)
+ def _select_contiguous(
+ self,
+ start: int,
+ length: int,
+ new_fingerprint: Optional[str] = None,
+ ) -> "Dataset":
+ """Create a new dataset with rows from a contiguous slice of data.
+ The slice is defined by that start index and its length.
+
+ Args:
+ start (`int`): start index.
+ length (`int`): length of the slice to select.
+ new_fingerprint (`str`, optional, default `None`): the new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds._select_contiguous(0, 4)
+ Dataset({
+ features: ['text', 'label'],
+ num_rows: 4
+ })
+ ```
+ """
+ if len(self.list_indexes()) > 0:
+ raise DatasetTransformationNotAllowedError(
+ "Using `.select` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it."
+ )
+
+ # If the array is empty we do nothing
+ if len(self) == 0:
+ return self
+
+ _check_valid_indices_value(start, len(self))
+ _check_valid_indices_value(start + length - 1, len(self))
+ if self._indices is None or length == 0:
+ return Dataset(
+ self.data.slice(start, length),
+ info=self.info.copy(),
+ split=self.split,
+ fingerprint=new_fingerprint,
+ )
+ else:
+ return Dataset(
+ self.data,
+ info=self.info.copy(),
+ split=self.split,
+ indices_table=self._indices.slice(start, length),
+ fingerprint=new_fingerprint,
+ )
+
+ @transmit_format
+ @fingerprint_transform(inplace=False, ignore_kwargs=["indices_cache_file_name"])
+ def _select_with_indices_mapping(
+ self,
+ indices: Iterable,
+ keep_in_memory: bool = False,
+ indices_cache_file_name: Optional[str] = None,
+ writer_batch_size: Optional[int] = 1000,
+ new_fingerprint: Optional[str] = None,
+ ) -> "Dataset":
+ """Create a new dataset with rows selected following the list/array of indices.
+ The new dataset is made by creating a new indices mapping on top of the main arrow table.
+
+ Args:
+ indices (sequence, iterable, range, ndarray or Series): List or 1D-array of integer indices for indexing.
+ keep_in_memory (`bool`, default `False`): Keep the indices mapping in memory instead of writing it to a cache file.
+ indices_cache_file_name (`str`, optional, default `None`): Provide the name of a path for the cache file. It is used to store the
+ indices mapping instead of the automatically generated cache file name.
+ writer_batch_size (`int`, default `1000`): Number of rows per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `.map()`.
+ new_fingerprint (`str`, optional, default `None`): the new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds._select_with_indices_mapping(range(4))
+ Dataset({
+ features: ['text', 'label'],
+ num_rows: 4
+ })
+ ```
+ """
+ if keep_in_memory and indices_cache_file_name is not None:
+ raise ValueError("Please use either `keep_in_memory` or `indices_cache_file_name` but not both.")
+
+ if len(self.list_indexes()) > 0:
+ raise DatasetTransformationNotAllowedError(
+ "Using `.select` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it."
+ )
+
+ # If the array is empty we do nothing
+ if len(self) == 0:
+ return self
+
+ # Prepare the writer for our indices arrow table
+ if keep_in_memory or indices_cache_file_name is None:
+ buf_writer = pa.BufferOutputStream()
+ tmp_file = None
+ writer = ArrowWriter(
+ stream=buf_writer, writer_batch_size=writer_batch_size, fingerprint=new_fingerprint, unit="indices"
+ )
+ else:
+ buf_writer = None
+ logger.info(f"Caching indices mapping at {indices_cache_file_name}")
+ cache_dir = os.path.dirname(indices_cache_file_name)
+ os.makedirs(cache_dir, exist_ok=True)
+ tmp_file = tempfile.NamedTemporaryFile("wb", dir=cache_dir, delete=False)
+ writer = ArrowWriter(
+ path=tmp_file.name, writer_batch_size=writer_batch_size, fingerprint=new_fingerprint, unit="indices"
+ )
+
+ indices = indices if isinstance(indices, list) else list(indices)
+
+ size = len(self)
+ if indices:
+ _check_valid_indices_value(int(max(indices)), size=size)
+ _check_valid_indices_value(int(min(indices)), size=size)
+ else:
+ return self._select_contiguous(0, 0, new_fingerprint=new_fingerprint)
+
+ indices_array = pa.array(indices, type=pa.uint64())
+ # Check if we need to convert indices
+ if self._indices is not None:
+ indices_array = self._indices.column(0).take(indices_array)
+
+ indices_table = pa.Table.from_arrays([indices_array], names=["indices"])
+
+ with writer:
+ try:
+ writer.write_table(indices_table)
+ writer.finalize() # close_stream=bool(buf_writer is None)) We only close if we are writing in a file
+ except (Exception, KeyboardInterrupt):
+ if tmp_file is not None:
+ tmp_file.close()
+ if os.path.exists(tmp_file.name):
+ os.remove(tmp_file.name)
+ raise
+
+ if tmp_file is not None:
+ tmp_file.close()
+ shutil.move(tmp_file.name, indices_cache_file_name)
+ umask = os.umask(0o666)
+ os.umask(umask)
+ os.chmod(indices_cache_file_name, 0o666 & ~umask)
+
+ # Return new Dataset object
+ if buf_writer is None:
+ return self._new_dataset_with_indices(
+ indices_cache_file_name=indices_cache_file_name, fingerprint=new_fingerprint
+ )
+ else:
+ return self._new_dataset_with_indices(indices_buffer=buf_writer.getvalue(), fingerprint=new_fingerprint)
+
+ def skip(self, n: int) -> "Dataset":
+ """
+ Create a new [`Dataset`] that skips the first `n` elements.
+
+ Args:
+ n (`int`):
+ Number of elements to skip.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
+ >>> list(ds.take(3))
+ [{'label': 1,
+ 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'},
+ {'label': 1,
+ 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'},
+ {'label': 1, 'text': 'effective but too-tepid biopic'}]
+ >>> ds = ds.skip(1)
+ >>> list(ds.take(3))
+ [{'label': 1,
+ 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'},
+ {'label': 1, 'text': 'effective but too-tepid biopic'},
+ {'label': 1,
+ 'text': 'if you sometimes like to go to the movies to have fun , wasabi is a good place to start .'}]
+ ```
+ """
+ return self.select(range(n, len(self)))
+
+ def repeat(self, num_times: int) -> "Dataset":
+ """
+ Create a new [`Dataset`] that repeats the underlying dataset `num_times` times.
+
+ Like itertools.repeat, repeating once just returns the full dataset.
+
+ Args:
+ num_times (`int`):
+ Number of times to repeat the dataset.
+
+ Example:
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
+ >>> ds = ds.take(2).repeat(2)
+ >>> list(ds)
+ [{'label': 1,
+ 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'},
+ {'label': 1,
+ 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'},
+ {'label': 1, 'text': 'effective but too-tepid biopic'},
+ {'label': 1,
+ 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'},
+ {'label': 1,
+ 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'},
+ {'label': 1, 'text': 'effective but too-tepid biopic'}]
+ ```
+ """
+ if num_times is None:
+ raise ValueError("Map style datasets do not support indefinite repetition.")
+ return _concatenate_map_style_datasets([self] * num_times) if num_times > 0 else self.select([])
+
+ def take(self, n: int) -> "Dataset":
+ """
+ Create a new [`Dataset`] with only the first `n` elements.
+
+ Args:
+ n (`int`):
+ Number of elements to take.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
+ >>> small_ds = ds.take(2)
+ >>> list(small_ds)
+ [{'label': 1,
+ 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'},
+ {'label': 1,
+ 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}]
+ ```
+ """
+ return self.select(range(n))
+
+ @transmit_format
+ @fingerprint_transform(inplace=False, ignore_kwargs=["load_from_cache_file", "indices_cache_file_name"])
+ def sort(
+ self,
+ column_names: Union[str, Sequence_[str]],
+ reverse: Union[bool, Sequence_[bool]] = False,
+ null_placement: str = "at_end",
+ keep_in_memory: bool = False,
+ load_from_cache_file: Optional[bool] = None,
+ indices_cache_file_name: Optional[str] = None,
+ writer_batch_size: Optional[int] = 1000,
+ new_fingerprint: Optional[str] = None,
+ ) -> "Dataset":
+ """Create a new dataset sorted according to a single or multiple columns.
+
+ Args:
+ column_names (`Union[str, Sequence[str]]`):
+ Column name(s) to sort by.
+ reverse (`Union[bool, Sequence[bool]]`, defaults to `False`):
+ If `True`, sort by descending order rather than ascending. If a single bool is provided,
+ the value is applied to the sorting of all column names. Otherwise a list of bools with the
+ same length and order as column_names must be provided.
+ null_placement (`str`, defaults to `at_end`):
+ Put `None` values at the beginning if `at_start` or `first` or at the end if `at_end` or `last`
+
+
+ keep_in_memory (`bool`, defaults to `False`):
+ Keep the sorted indices in memory instead of writing it to a cache file.
+ load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled):
+ If a cache file storing the sorted indices
+ can be identified, use it instead of recomputing.
+ indices_cache_file_name (`str`, *optional*, defaults to `None`):
+ Provide the name of a path for the cache file. It is used to store the
+ sorted indices instead of the automatically generated cache file name.
+ writer_batch_size (`int`, defaults to `1000`):
+ Number of rows per write operation for the cache file writer.
+ Higher value gives smaller cache files, lower value consume less temporary memory.
+ new_fingerprint (`str`, *optional*, defaults to `None`):
+ The new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset('cornell-movie-review-data/rotten_tomatoes', split='validation')
+ >>> ds['label'][:10]
+ [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
+ >>> sorted_ds = ds.sort('label')
+ >>> sorted_ds['label'][:10]
+ [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+ >>> another_sorted_ds = ds.sort(['label', 'text'], reverse=[True, False])
+ >>> another_sorted_ds['label'][:10]
+ [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
+ ```
+ """
+ if len(self.list_indexes()) > 0:
+ raise DatasetTransformationNotAllowedError(
+ "Using `.sort` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it."
+ )
+ # If the array is empty we do nothing
+ if len(self) == 0:
+ return self
+
+ # Check proper format of and for duplicates in column_names
+ if isinstance(column_names, str):
+ column_names = [column_names]
+
+ # Check proper format and length of reverse
+ if not isinstance(reverse, bool):
+ if len(reverse) != len(column_names):
+ raise ValueError(
+ "Parameter 'reverse' should be either a boolean or a list of booleans with the same length as 'column_names'."
+ )
+ else:
+ reverse = [reverse] * len(column_names)
+
+ # Check whether column name(s) exist in dataset
+ for column in column_names:
+ if not isinstance(column, str) or column not in self._data.column_names:
+ raise ValueError(
+ f"Column '{column}' not found in the dataset. Please provide a column selected in: {self._data.column_names}"
+ )
+
+ # Change null_placement to conform to pyarrow's sort_indices() while ensuring backwards compatibility
+ if null_placement not in ["at_start", "at_end"]:
+ if null_placement == "first":
+ null_placement = "at_start"
+ elif null_placement == "last":
+ null_placement = "at_end"
+ else:
+ raise ValueError(
+ f"null_placement '{null_placement}' is an invalid parameter value. Must be either 'last', 'at_end', 'first' or 'at_start'."
+ )
+
+ load_from_cache_file = load_from_cache_file if load_from_cache_file is not None else is_caching_enabled()
+
+ # Check if we've already cached this computation (indexed by a hash)
+ if self.cache_files:
+ if indices_cache_file_name is None:
+ # we create a unique hash from the function, current dataset file and the mapping args
+ indices_cache_file_name = self._get_cache_file_path(new_fingerprint)
+ if os.path.exists(indices_cache_file_name) and load_from_cache_file:
+ logger.info(f"Loading cached sorted indices for dataset at {indices_cache_file_name}")
+ return self._new_dataset_with_indices(
+ fingerprint=new_fingerprint, indices_cache_file_name=indices_cache_file_name
+ )
+
+ sort_table = query_table(
+ table=self._data,
+ key=slice(0, len(self)),
+ indices=self._indices,
+ )
+
+ sort_keys = [
+ (col, "ascending" if not col_reverse else "descending") for col, col_reverse in zip(column_names, reverse)
+ ]
+
+ indices = pc.sort_indices(sort_table, sort_keys=sort_keys, null_placement=null_placement)
+
+ return self.select(
+ indices=indices,
+ keep_in_memory=keep_in_memory,
+ indices_cache_file_name=indices_cache_file_name,
+ writer_batch_size=writer_batch_size,
+ new_fingerprint=new_fingerprint,
+ )
+
+ @transmit_format
+ @fingerprint_transform(
+ inplace=False, randomized_function=True, ignore_kwargs=["load_from_cache_file", "indices_cache_file_name"]
+ )
+ def shuffle(
+ self,
+ seed: Optional[int] = None,
+ generator: Optional[np.random.Generator] = None,
+ keep_in_memory: bool = False,
+ load_from_cache_file: Optional[bool] = None,
+ indices_cache_file_name: Optional[str] = None,
+ writer_batch_size: Optional[int] = 1000,
+ new_fingerprint: Optional[str] = None,
+ ) -> "Dataset":
+ """Create a new Dataset where the rows are shuffled.
+
+ Currently shuffling uses numpy random generators.
+ You can either supply a NumPy BitGenerator to use, or a seed to initiate NumPy's default random generator (PCG64).
+
+ Shuffling takes the list of indices `[0:len(my_dataset)]` and shuffles it to create an indices mapping.
+ However as soon as your [`Dataset`] has an indices mapping, the speed can become 10x slower.
+ This is because there is an extra step to get the row index to read using the indices mapping, and most importantly, you aren't reading contiguous chunks of data anymore.
+ To restore the speed, you'd need to rewrite the entire dataset on your disk again using [`Dataset.flatten_indices`], which removes the indices mapping.
+ This may take a lot of time depending of the size of your dataset though:
+
+ ```python
+ my_dataset[0] # fast
+ my_dataset = my_dataset.shuffle(seed=42)
+ my_dataset[0] # up to 10x slower
+ my_dataset = my_dataset.flatten_indices() # rewrite the shuffled dataset on disk as contiguous chunks of data
+ my_dataset[0] # fast again
+ ```
+
+ In this case, we recommend switching to an [`IterableDataset`] and leveraging its fast approximate shuffling method [`IterableDataset.shuffle`].
+ It only shuffles the shards order and adds a shuffle buffer to your dataset, which keeps the speed of your dataset optimal:
+
+ ```python
+ my_iterable_dataset = my_dataset.to_iterable_dataset(num_shards=128)
+ for example in enumerate(my_iterable_dataset): # fast
+ pass
+
+ shuffled_iterable_dataset = my_iterable_dataset.shuffle(seed=42, buffer_size=100)
+
+ for example in enumerate(shuffled_iterable_dataset): # as fast as before
+ pass
+ ```
+
+ Args:
+ seed (`int`, *optional*):
+ A seed to initialize the default BitGenerator if `generator=None`.
+ If `None`, then fresh, unpredictable entropy will be pulled from the OS.
+ If an `int` or `array_like[ints]` is passed, then it will be passed to SeedSequence to derive the initial BitGenerator state.
+ generator (`numpy.random.Generator`, *optional*):
+ Numpy random Generator to use to compute the permutation of the dataset rows.
+ If `generator=None` (default), uses `np.random.default_rng` (the default BitGenerator (PCG64) of NumPy).
+ keep_in_memory (`bool`, default `False`):
+ Keep the shuffled indices in memory instead of writing it to a cache file.
+ load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled):
+ If a cache file storing the shuffled indices
+ can be identified, use it instead of recomputing.
+ indices_cache_file_name (`str`, *optional*):
+ Provide the name of a path for the cache file. It is used to store the
+ shuffled indices instead of the automatically generated cache file name.
+ writer_batch_size (`int`, defaults to `1000`):
+ Number of rows per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`.
+ new_fingerprint (`str`, *optional*, defaults to `None`):
+ The new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds['label'][:10]
+ [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
+
+ # set a seed
+ >>> shuffled_ds = ds.shuffle(seed=42)
+ >>> shuffled_ds['label'][:10]
+ [1, 0, 1, 1, 0, 0, 0, 0, 0, 0]
+ ```
+ """
+ if len(self.list_indexes()) > 0:
+ raise DatasetTransformationNotAllowedError(
+ "Using `.shuffle` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it."
+ )
+ # If the array is empty we do nothing
+ if len(self) == 0:
+ return self
+
+ if keep_in_memory and indices_cache_file_name is not None:
+ raise ValueError("Please use either `keep_in_memory` or `indices_cache_file_name` but not both.")
+
+ if seed is not None and generator is not None:
+ raise ValueError("Both `seed` and `generator` were provided. Please specify just one of them.")
+
+ if generator is not None and not isinstance(generator, np.random.Generator):
+ raise ValueError("The provided generator must be an instance of numpy.random.Generator")
+
+ load_from_cache_file = load_from_cache_file if load_from_cache_file is not None else is_caching_enabled()
+
+ if generator is None:
+ if seed is None:
+ _, seed, pos, *_ = np.random.get_state()
+ seed = seed[pos] if pos < 624 else seed[0]
+ _ = np.random.random() # do 1 step of rng
+ generator = np.random.default_rng(seed)
+
+ # Check if we've already cached this computation (indexed by a hash)
+ if self.cache_files:
+ if indices_cache_file_name is None:
+ # we create a unique hash from the function, current dataset file and the mapping args
+ indices_cache_file_name = self._get_cache_file_path(new_fingerprint)
+ if os.path.exists(indices_cache_file_name) and load_from_cache_file:
+ logger.info(f"Loading cached shuffled indices for dataset at {indices_cache_file_name}")
+ return self._new_dataset_with_indices(
+ fingerprint=new_fingerprint, indices_cache_file_name=indices_cache_file_name
+ )
+
+ permutation = generator.permutation(len(self))
+
+ return self.select(
+ indices=permutation,
+ keep_in_memory=keep_in_memory,
+ indices_cache_file_name=indices_cache_file_name if not keep_in_memory else None,
+ writer_batch_size=writer_batch_size,
+ new_fingerprint=new_fingerprint,
+ )
+
+ @transmit_format
+ @fingerprint_transform(
+ inplace=False,
+ randomized_function=True,
+ fingerprint_names=["train_new_fingerprint", "test_new_fingerprint"],
+ ignore_kwargs=["load_from_cache_file", "train_indices_cache_file_name", "test_indices_cache_file_name"],
+ )
+ def train_test_split(
+ self,
+ test_size: Union[float, int, None] = None,
+ train_size: Union[float, int, None] = None,
+ shuffle: bool = True,
+ stratify_by_column: Optional[str] = None,
+ seed: Optional[int] = None,
+ generator: Optional[np.random.Generator] = None,
+ keep_in_memory: bool = False,
+ load_from_cache_file: Optional[bool] = None,
+ train_indices_cache_file_name: Optional[str] = None,
+ test_indices_cache_file_name: Optional[str] = None,
+ writer_batch_size: Optional[int] = 1000,
+ train_new_fingerprint: Optional[str] = None,
+ test_new_fingerprint: Optional[str] = None,
+ ) -> "DatasetDict":
+ """Return a dictionary ([`datasets.DatasetDict`]) with two random train and test subsets (`train` and `test` `Dataset` splits).
+ Splits are created from the dataset according to `test_size`, `train_size` and `shuffle`.
+
+ This method is similar to scikit-learn `train_test_split`.
+
+ Args:
+ test_size (`Union[float, int, None]`, *optional*):
+ Size of the test split
+ If `float`, should be between `0.0` and `1.0` and represent the proportion of the dataset to include in the test split.
+ If `int`, represents the absolute number of test samples.
+ If `None`, the value is set to the complement of the train size.
+ If `train_size` is also `None`, it will be set to `0.25`.
+ train_size (`Union[float, int, None]`, *optional*):
+ Size of the train split
+ If `float`, should be between `0.0` and `1.0` and represent the proportion of the dataset to include in the train split.
+ If `int`, represents the absolute number of train samples.
+ If `None`, the value is automatically set to the complement of the test size.
+ shuffle (`bool`, *optional*, defaults to `True`):
+ Whether or not to shuffle the data before splitting.
+ stratify_by_column (`str`, *optional*, defaults to `None`):
+ The column name of labels to be used to perform stratified split of data.
+ seed (`int`, *optional*):
+ A seed to initialize the default BitGenerator if `generator=None`.
+ If `None`, then fresh, unpredictable entropy will be pulled from the OS.
+ If an `int` or `array_like[ints]` is passed, then it will be passed to SeedSequence to derive the initial BitGenerator state.
+ generator (`numpy.random.Generator`, *optional*):
+ Numpy random Generator to use to compute the permutation of the dataset rows.
+ If `generator=None` (default), uses `np.random.default_rng` (the default BitGenerator (PCG64) of NumPy).
+ keep_in_memory (`bool`, defaults to `False`):
+ Keep the splits indices in memory instead of writing it to a cache file.
+ load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled):
+ If a cache file storing the splits indices
+ can be identified, use it instead of recomputing.
+ train_cache_file_name (`str`, *optional*):
+ Provide the name of a path for the cache file. It is used to store the
+ train split indices instead of the automatically generated cache file name.
+ test_cache_file_name (`str`, *optional*):
+ Provide the name of a path for the cache file. It is used to store the
+ test split indices instead of the automatically generated cache file name.
+ writer_batch_size (`int`, defaults to `1000`):
+ Number of rows per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`.
+ train_new_fingerprint (`str`, *optional*, defaults to `None`):
+ The new fingerprint of the train set after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments
+ test_new_fingerprint (`str`, *optional*, defaults to `None`):
+ The new fingerprint of the test set after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds = ds.train_test_split(test_size=0.2, shuffle=True)
+ DatasetDict({
+ train: Dataset({
+ features: ['text', 'label'],
+ num_rows: 852
+ })
+ test: Dataset({
+ features: ['text', 'label'],
+ num_rows: 214
+ })
+ })
+
+ # set a seed
+ >>> ds = ds.train_test_split(test_size=0.2, seed=42)
+
+ # stratified split
+ >>> ds = load_dataset("stanfordnlp/imdb",split="train")
+ Dataset({
+ features: ['text', 'label'],
+ num_rows: 25000
+ })
+ >>> ds = ds.train_test_split(test_size=0.2, stratify_by_column="label")
+ DatasetDict({
+ train: Dataset({
+ features: ['text', 'label'],
+ num_rows: 20000
+ })
+ test: Dataset({
+ features: ['text', 'label'],
+ num_rows: 5000
+ })
+ })
+ ```
+ """
+ from .dataset_dict import DatasetDict # import here because of circular dependency
+
+ if len(self.list_indexes()) > 0:
+ raise DatasetTransformationNotAllowedError(
+ "Using `.train_test_split` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it."
+ )
+ # If the array is empty we do nothing
+ if len(self) == 0:
+ return DatasetDict({"train": self, "test": self})
+
+ if test_size is None and train_size is None:
+ test_size = 0.25
+
+ # Safety checks similar to scikit-learn's ones.
+ # (adapted from https://github.com/scikit-learn/scikit-learn/blob/fd237278e895b42abe8d8d09105cbb82dc2cbba7/sklearn/model_selection/_split.py#L1750)
+ n_samples = len(self)
+ if (
+ isinstance(test_size, int)
+ and (test_size >= n_samples or test_size <= 0)
+ or isinstance(test_size, float)
+ and (test_size <= 0 or test_size >= 1)
+ ):
+ raise ValueError(
+ f"test_size={test_size} should be either positive and smaller "
+ f"than the number of samples {n_samples} or a float in the (0, 1) range"
+ )
+
+ if (
+ isinstance(train_size, int)
+ and (train_size >= n_samples or train_size <= 0)
+ or isinstance(train_size, float)
+ and (train_size <= 0 or train_size >= 1)
+ ):
+ raise ValueError(
+ f"train_size={train_size} should be either positive and smaller "
+ f"than the number of samples {n_samples} or a float in the (0, 1) range"
+ )
+
+ if train_size is not None and not isinstance(train_size, (int, float)):
+ raise ValueError(f"Invalid value for train_size: {train_size} of type {type(train_size)}")
+ if test_size is not None and not isinstance(test_size, (int, float)):
+ raise ValueError(f"Invalid value for test_size: {test_size} of type {type(test_size)}")
+
+ if isinstance(train_size, float) and isinstance(test_size, float) and train_size + test_size > 1:
+ raise ValueError(
+ f"The sum of test_size and train_size = {train_size + test_size}, should be in the (0, 1)"
+ " range. Reduce test_size and/or train_size."
+ )
+
+ if isinstance(test_size, float):
+ n_test = ceil(test_size * n_samples)
+ elif isinstance(test_size, int):
+ n_test = float(test_size)
+
+ if isinstance(train_size, float):
+ n_train = floor(train_size * n_samples)
+ elif isinstance(train_size, int):
+ n_train = float(train_size)
+
+ if train_size is None:
+ n_train = n_samples - n_test
+ elif test_size is None:
+ n_test = n_samples - n_train
+
+ if n_train + n_test > n_samples:
+ raise ValueError(
+ f"The sum of train_size and test_size = {n_train + n_test}, "
+ "should be smaller than the number of "
+ f"samples {n_samples}. Reduce test_size and/or "
+ "train_size."
+ )
+
+ n_train, n_test = int(n_train), int(n_test)
+
+ if n_train == 0:
+ raise ValueError(
+ f"With n_samples={n_samples}, test_size={test_size} and train_size={train_size}, the "
+ "resulting train set will be empty. Adjust any of the "
+ "aforementioned parameters."
+ )
+
+ load_from_cache_file = load_from_cache_file if load_from_cache_file is not None else is_caching_enabled()
+
+ if generator is None and shuffle is True:
+ if seed is None:
+ _, seed, pos, *_ = np.random.get_state()
+ seed = seed[pos] if pos < 624 else seed[0]
+ _ = np.random.random() # do 1 step of rng
+ generator = np.random.default_rng(seed)
+
+ # Check if we've already cached this computation (indexed by a hash)
+ if self.cache_files:
+ if train_indices_cache_file_name is None or test_indices_cache_file_name is None:
+ # we create a unique hash from the function, current dataset file and the mapping args
+
+ if train_indices_cache_file_name is None:
+ train_indices_cache_file_name = self._get_cache_file_path(train_new_fingerprint)
+ if test_indices_cache_file_name is None:
+ test_indices_cache_file_name = self._get_cache_file_path(test_new_fingerprint)
+ if (
+ os.path.exists(train_indices_cache_file_name)
+ and os.path.exists(test_indices_cache_file_name)
+ and load_from_cache_file
+ ):
+ logger.info(
+ f"Loading cached split indices for dataset at {train_indices_cache_file_name} and {test_indices_cache_file_name}"
+ )
+ return DatasetDict(
+ {
+ "train": self._new_dataset_with_indices(
+ fingerprint=train_new_fingerprint, indices_cache_file_name=train_indices_cache_file_name
+ ),
+ "test": self._new_dataset_with_indices(
+ fingerprint=test_new_fingerprint, indices_cache_file_name=test_indices_cache_file_name
+ ),
+ }
+ )
+ if not shuffle:
+ if stratify_by_column is not None:
+ raise ValueError("Stratified train/test split is not implemented for `shuffle=False`")
+ train_indices = np.arange(n_train)
+ test_indices = np.arange(n_train, n_train + n_test)
+ else:
+ # stratified partition
+ if stratify_by_column is not None:
+ if stratify_by_column not in self._info.features.keys():
+ raise ValueError(f"Key {stratify_by_column} not found in {self._info.features.keys()}")
+ if not isinstance(self._info.features[stratify_by_column], ClassLabel):
+ raise ValueError(
+ f"Stratifying by column is only supported for {ClassLabel.__name__} column, and column {stratify_by_column} is {type(self._info.features[stratify_by_column]).__name__}."
+ )
+ try:
+ train_indices, test_indices = next(
+ stratified_shuffle_split_generate_indices(
+ np.asarray(self.with_format("numpy")[stratify_by_column]), n_train, n_test, rng=generator
+ )
+ )
+ except Exception as error:
+ if str(error) == "Minimum class count error":
+ raise ValueError(
+ f"The least populated class in {stratify_by_column} column has only 1"
+ " member, which is too few. The minimum"
+ " number of groups for any class cannot"
+ " be less than 2."
+ )
+ else:
+ raise error
+
+ # random partition
+ else:
+ permutation = generator.permutation(len(self))
+ test_indices = permutation[:n_test]
+ train_indices = permutation[n_test : (n_test + n_train)]
+
+ train_split = self.select(
+ indices=train_indices,
+ keep_in_memory=keep_in_memory,
+ indices_cache_file_name=train_indices_cache_file_name,
+ writer_batch_size=writer_batch_size,
+ new_fingerprint=train_new_fingerprint,
+ )
+ test_split = self.select(
+ indices=test_indices,
+ keep_in_memory=keep_in_memory,
+ indices_cache_file_name=test_indices_cache_file_name,
+ writer_batch_size=writer_batch_size,
+ new_fingerprint=test_new_fingerprint,
+ )
+
+ return DatasetDict({"train": train_split, "test": test_split})
+
+ def shard(
+ self,
+ num_shards: int,
+ index: int,
+ contiguous: bool = True,
+ keep_in_memory: bool = False,
+ indices_cache_file_name: Optional[str] = None,
+ writer_batch_size: Optional[int] = 1000,
+ ) -> "Dataset":
+ """Return the `index`-nth shard from dataset split into `num_shards` pieces.
+
+ This shards deterministically. `dataset.shard(n, i)` splits the dataset into contiguous chunks,
+ so it can be easily concatenated back together after processing. If `len(dataset) % n == l`, then the
+ first `l` dataset each have length `(len(dataset) // n) + 1`, and the remaining dataset have length `(len(dataset) // n)`.
+ `datasets.concatenate_datasets([dset.shard(n, i) for i in range(n)])` returns a dataset with the same order as the original.
+
+ Note: n should be less or equal to the number of elements in the dataset `len(dataset)`.
+
+ On the other hand, `dataset.shard(n, i, contiguous=False)` contains all elements of the dataset whose index mod `n = i`.
+
+ Be sure to shard before using any randomizing operator (such as `shuffle`).
+ It is best if the shard operator is used early in the dataset pipeline.
+
+ Args:
+ num_shards (`int`):
+ How many shards to split the dataset into.
+ index (`int`):
+ Which shard to select and return.
+ contiguous: (`bool`, defaults to `True`):
+ Whether to select contiguous blocks of indices for shards.
+ keep_in_memory (`bool`, defaults to `False`):
+ Keep the dataset in memory instead of writing it to a cache file.
+ indices_cache_file_name (`str`, *optional*):
+ Provide the name of a path for the cache file. It is used to store the
+ indices of each shard instead of the automatically generated cache file name.
+ writer_batch_size (`int`, defaults to `1000`):
+ This only concerns the indices mapping.
+ Number of indices per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds
+ Dataset({
+ features: ['text', 'label'],
+ num_rows: 1066
+ })
+ >>> ds = ds.shard(num_shards=2, index=0)
+ >>> ds
+ Dataset({
+ features: ['text', 'label'],
+ num_rows: 533
+ })
+ ```
+ """
+ if not 0 <= index < num_shards:
+ raise ValueError("index should be in [0, num_shards-1]")
+ if contiguous:
+ div = len(self) // num_shards
+ mod = len(self) % num_shards
+ start = div * index + min(index, mod)
+ end = start + div + (1 if index < mod else 0)
+ indices = range(start, end)
+ else:
+ indices = np.arange(index, len(self), num_shards)
+
+ return self.select(
+ indices=indices,
+ keep_in_memory=keep_in_memory,
+ indices_cache_file_name=indices_cache_file_name,
+ writer_batch_size=writer_batch_size,
+ )
+
+ def to_csv(
+ self,
+ path_or_buf: Union[PathLike, BinaryIO],
+ batch_size: Optional[int] = None,
+ num_proc: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ **to_csv_kwargs,
+ ) -> int:
+ """Exports the dataset to csv
+
+ Args:
+ path_or_buf (`PathLike` or `FileOrBuffer`):
+ Either a path to a file (e.g. `file.csv`), a remote URI (e.g. `hf://datasets/username/my_dataset_name/data.csv`),
+ or a BinaryIO, where the dataset will be saved to in the specified format.
+ batch_size (`int`, *optional*):
+ Size of the batch to load in memory and write at once.
+ Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+ num_proc (`int`, *optional*):
+ Number of processes for multiprocessing. By default it doesn't
+ use multiprocessing. `batch_size` in this case defaults to
+ `datasets.config.DEFAULT_MAX_BATCH_SIZE` but feel free to make it 5x or 10x of the default
+ value if you have sufficient compute power.
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+
+
+ **to_csv_kwargs (additional keyword arguments):
+ Parameters to pass to pandas's [`pandas.DataFrame.to_csv`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_csv.html).
+
+
+
+ Now, `index` defaults to `False` if not specified.
+
+ If you would like to write the index, pass `index=True` and also set a name for the index column by
+ passing `index_label`.
+
+
+
+ Returns:
+ `int`: The number of characters or bytes written.
+
+ Example:
+
+ ```py
+ >>> ds.to_csv("path/to/dataset/directory")
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.csv import CsvDatasetWriter
+
+ return CsvDatasetWriter(
+ self,
+ path_or_buf,
+ batch_size=batch_size,
+ num_proc=num_proc,
+ storage_options=storage_options,
+ **to_csv_kwargs,
+ ).write()
+
+ def to_dict(self, batch_size: Optional[int] = None, batched: bool = False) -> Union[dict, Iterator[dict]]:
+ """Returns the dataset as a Python dict. Can also return a generator for large datasets.
+
+ Args:
+ batch_size (`int`, *optional*): The size (number of rows) of the batches if `batched` is `True`.
+ Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+ batched (`bool`):
+ Set to `True` to return a generator that yields the dataset as batches
+ of `batch_size` rows. Defaults to `False` (returns the whole datasets once).
+
+ Returns:
+ `dict` or `Iterator[dict]`
+
+ Example:
+
+ ```py
+ >>> ds.to_dict()
+ ```
+ """
+ result = query_table(
+ table=self._data,
+ key=slice(0, len(self)),
+ indices=self._indices,
+ ).to_pydict()
+ from .utils.json import get_json_field_paths_from_feature, json_decode_field
+
+ for json_field_path in get_json_field_paths_from_feature(self.features):
+ col, *json_field_subpath = json_field_path
+ result[col] = [json_decode_field(row, json_field_subpath) for row in result[col]]
+ return result
+
+ def to_list(self) -> list:
+ """Returns the dataset as a Python list.
+
+ Returns:
+ `list`
+
+ Example:
+
+ ```py
+ >>> ds.to_list()
+ ```
+ """
+ result = query_table(
+ table=self._data,
+ key=slice(0, len(self)),
+ indices=self._indices,
+ ).to_pylist()
+ from .utils.json import get_json_field_paths_from_feature, json_decode_field
+
+ for json_field_path in get_json_field_paths_from_feature(self.features):
+ col, *json_field_subpath = json_field_path
+ for row in result:
+ row[col] = json_decode_field(row[col], json_field_subpath)
+ return result
+
+ def to_json(
+ self,
+ path_or_buf: Union[PathLike, BinaryIO],
+ batch_size: Optional[int] = None,
+ num_proc: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ **to_json_kwargs,
+ ) -> int:
+ """Export the dataset to JSON Lines or JSON.
+
+ The default output format is [JSON Lines](https://jsonlines.org/).
+ To export to [JSON](https://www.json.org), pass `lines=False` argument and the desired `orient`.
+
+ Args:
+ path_or_buf (`PathLike` or `FileOrBuffer`):
+ Either a path to a file (e.g. `file.json`), a remote URI (e.g. `hf://datasets/username/my_dataset_name/data.json`),
+ or a BinaryIO, where the dataset will be saved to in the specified format.
+ batch_size (`int`, *optional*):
+ Size of the batch to load in memory and write at once.
+ Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+ num_proc (`int`, *optional*):
+ Number of processes for multiprocessing. By default, it doesn't
+ use multiprocessing. `batch_size` in this case defaults to
+ `datasets.config.DEFAULT_MAX_BATCH_SIZE` but feel free to make it 5x or 10x of the default
+ value if you have sufficient compute power.
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+
+
+ **to_json_kwargs (additional keyword arguments):
+ Parameters to pass to pandas's [`pandas.DataFrame.to_json`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_json.html).
+ Default arguments are `lines=True` and `orient="records".
+
+
+
+ The parameter `index` defaults to `False` if `orient` is `"split"` or `"table"`.
+
+ If you would like to write the index, pass `index=True`.
+
+
+
+ Returns:
+ `int`: The number of characters or bytes written.
+
+ Example:
+
+ ```py
+ >>> ds.to_json("path/to/dataset/directory/filename.jsonl")
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.json import JsonDatasetWriter
+
+ return JsonDatasetWriter(
+ self,
+ path_or_buf,
+ batch_size=batch_size,
+ num_proc=num_proc,
+ storage_options=storage_options,
+ **to_json_kwargs,
+ ).write()
+
+ def to_pandas(
+ self, batch_size: Optional[int] = None, batched: bool = False
+ ) -> Union[pd.DataFrame, Iterator[pd.DataFrame]]:
+ """Returns the dataset as a `pandas.DataFrame`. Can also return a generator for large datasets.
+
+ Args:
+ batch_size (`int`, *optional*):
+ The size (number of rows) of the batches if `batched` is `True`.
+ Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+ batched (`bool`):
+ Set to `True` to return a generator that yields the dataset as batches
+ of `batch_size` rows. Defaults to `False` (returns the whole datasets once).
+
+ Returns:
+ `pandas.DataFrame` or `Iterator[pandas.DataFrame]`
+
+ Example:
+
+ ```py
+ >>> ds.to_pandas()
+ ```
+ """
+ if not batched:
+ return query_table(
+ table=self._data,
+ key=slice(0, len(self)),
+ indices=self._indices,
+ ).to_pandas(types_mapper=pandas_types_mapper)
+ else:
+ batch_size = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
+ return (
+ query_table(
+ table=self._data,
+ key=slice(offset, offset + batch_size),
+ indices=self._indices,
+ ).to_pandas(types_mapper=pandas_types_mapper)
+ for offset in range(0, len(self), batch_size)
+ )
+
+ def to_polars(
+ self,
+ batch_size: Optional[int] = None,
+ batched: bool = False,
+ schema_overrides: Optional[dict] = None,
+ rechunk: bool = True,
+ ) -> Union["pl.DataFrame", Iterator["pl.DataFrame"]]:
+ """Returns the dataset as a `polars.DataFrame`. Can also return a generator for large datasets.
+
+ Args:
+ batch_size (`int`, *optional*):
+ The size (number of rows) of the batches if `batched` is `True`.
+ Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+ batched (`bool`):
+ Set to `True` to return a generator that yields the dataset as batches
+ of `batch_size` rows. Defaults to `False` (returns the whole datasets once).
+ schema_overrides (`dict`, *optional*):
+ Support type specification or override of one or more columns; note that
+ any dtypes inferred from the schema param will be overridden.
+ rechunk (`bool`):
+ Make sure that all data is in contiguous memory. Defaults to `True`.
+ Returns:
+ `polars.DataFrame` or `Iterator[polars.DataFrame]`
+
+ Example:
+
+ ```py
+ >>> ds.to_polars()
+ ```
+ """
+ if config.POLARS_AVAILABLE:
+ import polars as pl
+
+ if not batched:
+ return pl.from_arrow(
+ query_table(
+ table=self._data,
+ key=slice(0, len(self)),
+ indices=self._indices if self._indices is not None else None,
+ ),
+ schema_overrides=schema_overrides,
+ rechunk=rechunk,
+ )
+ else:
+ batch_size = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
+ return (
+ pl.from_arrow(
+ query_table(
+ table=self._data,
+ key=slice(offset, offset + batch_size),
+ indices=self._indices if self._indices is not None else None,
+ ),
+ schema_overrides=schema_overrides,
+ rechunk=rechunk,
+ )
+ for offset in range(0, len(self), batch_size)
+ )
+ else:
+ raise ValueError("Polars needs to be installed to be able to return Polars dataframes.")
+
+ def to_parquet(
+ self,
+ path_or_buf: Union[PathLike, BinaryIO],
+ batch_size: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ **parquet_writer_kwargs,
+ ) -> int:
+ """Exports the dataset to parquet
+
+ Args:
+ path_or_buf (`PathLike` or `FileOrBuffer`):
+ Either a path to a file (e.g. `file.parquet`), a remote URI (e.g. `hf://datasets/username/my_dataset_name/data.parquet`),
+ or a BinaryIO, where the dataset will be saved to in the specified format.
+ batch_size (`int`, *optional*):
+ Size of the batch to load in memory and write at once.
+ By default it aims for row groups with maximum uncompressed byte size of "100MB",
+ defined by `datasets.config.MAX_ROW_GROUP_SIZE`.
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+
+
+ **parquet_writer_kwargs (additional keyword arguments):
+ Parameters to pass to PyArrow's `pyarrow.parquet.ParquetWriter`.
+
+ Returns:
+ `int`: The number of characters or bytes written.
+
+ Example:
+
+ ```py
+ >>> ds.to_parquet("path/to/dataset/directory")
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.parquet import ParquetDatasetWriter
+
+ return ParquetDatasetWriter(
+ self, path_or_buf, batch_size=batch_size, storage_options=storage_options, **parquet_writer_kwargs
+ ).write()
+
+ def to_sql(
+ self,
+ name: str,
+ con: Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"],
+ batch_size: Optional[int] = None,
+ num_proc: Optional[int] = None,
+ **sql_writer_kwargs,
+ ) -> int:
+ """Exports the dataset to a SQL database.
+
+ Args:
+ name (`str`):
+ Name of SQL table.
+ con (`str` or `sqlite3.Connection` or `sqlalchemy.engine.Connection` or `sqlalchemy.engine.Connection`):
+ A [URI string](https://docs.sqlalchemy.org/en/13/core/engines.html#database-urls) or a SQLite3/SQLAlchemy connection object used to write to a database.
+ batch_size (`int`, *optional*):
+ Size of the batch to load in memory and write at once.
+ Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+ num_proc (`int`, *optional*):
+ Number of processes for multiprocessing. By default, it doesn't
+ use multiprocessing. `batch_size` in this case defaults to
+ `datasets.config.DEFAULT_MAX_BATCH_SIZE` but feel free to make it 5x or 10x of the default
+ value if you have sufficient compute power.
+ **sql_writer_kwargs (additional keyword arguments):
+ Parameters to pass to pandas's [`pandas.DataFrame.to_sql`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_sql.html).
+
+
+
+ Now, `index` defaults to `False` if not specified.
+
+ If you would like to write the index, pass `index=True` and also set a name for the index column by
+ passing `index_label`.
+
+
+
+ Returns:
+ `int`: The number of records written.
+
+ Example:
+
+ ```py
+ >>> # con provided as a connection URI string
+ >>> ds.to_sql("data", "sqlite:///my_own_db.sql")
+ >>> # con provided as a sqlite3 connection object
+ >>> import sqlite3
+ >>> con = sqlite3.connect("my_own_db.sql")
+ >>> with con:
+ ... ds.to_sql("data", con)
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.sql import SqlDatasetWriter
+
+ return SqlDatasetWriter(self, name, con, batch_size=batch_size, num_proc=num_proc, **sql_writer_kwargs).write()
+
+ def _estimate_nbytes(self) -> int:
+ dataset_nbytes = self.data.nbytes
+
+ # Find decodable columns, because if there are any, we need to
+ # adjust the dataset size computation (needed for sharding) to account for possible external files
+ decodable_columns = [
+ k for k, v in self._info.features.items() if require_decoding(v, ignore_decode_attribute=True)
+ ]
+
+ if decodable_columns:
+ # Approximate the space needed to store the bytes from the external files by analyzing the first 1000 examples
+ extra_nbytes = 0
+
+ def extra_nbytes_visitor(array, feature):
+ nonlocal extra_nbytes
+ if isinstance(feature, (Audio, Image, Video)):
+ for x in array.to_pylist():
+ if x is not None and x["bytes"] is None and x["path"] is not None:
+ size = xgetsize(x["path"])
+ extra_nbytes += size
+ extra_nbytes -= array.field("path").nbytes
+
+ table = self.with_format("arrow")[:1000]
+ table_visitor(table, extra_nbytes_visitor)
+
+ extra_nbytes = extra_nbytes * len(self.data) // len(table)
+ dataset_nbytes = dataset_nbytes + extra_nbytes
+
+ if self._indices is not None:
+ dataset_nbytes = dataset_nbytes * len(self._indices) // len(self.data)
+ return dataset_nbytes
+
+ @staticmethod
+ def _generate_tables_from_shards(shards: list["Dataset"], batch_size: int):
+ for shard_idx, shard in enumerate(shards):
+ for pa_table in shard.with_format("arrow").iter(batch_size):
+ yield shard_idx, pa_table
+
+ @staticmethod
+ def _generate_tables_from_cache_file(filename: str):
+ for batch_idx, batch in enumerate(_memory_mapped_record_batch_reader_from_file(filename)):
+ yield batch_idx, pa.Table.from_batches([batch])
+
+ def to_iterable_dataset(self, num_shards: Optional[int] = 1) -> "IterableDataset":
+ """Get an [`datasets.IterableDataset`] from a map-style [`datasets.Dataset`].
+ This is equivalent to loading a dataset in streaming mode with [`datasets.load_dataset`], but much faster since the data is streamed from local files.
+
+ Contrary to map-style datasets, iterable datasets are lazy and can only be iterated over (e.g. using a for loop).
+ Since they are read sequentially in training loops, iterable datasets are much faster than map-style datasets.
+ All the transformations applied to iterable datasets like filtering or processing are done on-the-fly when you start iterating over the dataset.
+
+ Still, it is possible to shuffle an iterable dataset using [`datasets.IterableDataset.shuffle`].
+ This is a fast approximate shuffling that works best if you have multiple shards and if you specify a buffer size that is big enough.
+
+ To get the best speed performance, make sure your dataset doesn't have an indices mapping.
+ If this is the case, the data are not read contiguously, which can be slow sometimes.
+ You can use `ds = ds.flatten_indices()` to write your dataset in contiguous chunks of data and have optimal speed before switching to an iterable dataset.
+
+ Args:
+ num_shards (`int`, default to `1`):
+ Number of shards to define when instantiating the iterable dataset. This is especially useful for big datasets to be able to shuffle properly,
+ and also to enable fast parallel loading using a PyTorch DataLoader or in distributed setups for example.
+ Shards are defined using [`datasets.Dataset.shard`]: it simply slices the data without writing anything on disk.
+
+ Returns:
+ [`datasets.IterableDataset`]
+
+ Example:
+
+ Basic usage:
+ ```python
+ >>> ids = ds.to_iterable_dataset()
+ >>> for example in ids:
+ ... pass
+ ```
+
+ With lazy filtering and processing:
+ ```python
+ >>> ids = ds.to_iterable_dataset()
+ >>> ids = ids.filter(filter_fn).map(process_fn) # will filter and process on-the-fly when you start iterating over the iterable dataset
+ >>> for example in ids:
+ ... pass
+ ```
+
+ With sharding to enable efficient shuffling:
+ ```python
+ >>> ids = ds.to_iterable_dataset(num_shards=64) # the dataset is split into 64 shards to be iterated over
+ >>> ids = ids.shuffle(buffer_size=10_000) # will shuffle the shards order and use a shuffle buffer for fast approximate shuffling when you start iterating
+ >>> for example in ids:
+ ... pass
+ ```
+
+ With a PyTorch DataLoader:
+ ```python
+ >>> import torch
+ >>> ids = ds.to_iterable_dataset(num_shards=64)
+ >>> ids = ids.filter(filter_fn).map(process_fn)
+ >>> dataloader = torch.utils.data.DataLoader(ids, num_workers=4) # will assign 64 / 4 = 16 shards to each worker to load, filter and process when you start iterating
+ >>> for example in ids:
+ ... pass
+ ```
+
+ With a PyTorch DataLoader and shuffling:
+ ```python
+ >>> import torch
+ >>> ids = ds.to_iterable_dataset(num_shards=64)
+ >>> ids = ids.shuffle(buffer_size=10_000) # will shuffle the shards order and use a shuffle buffer when you start iterating
+ >>> dataloader = torch.utils.data.DataLoader(ids, num_workers=4) # will assign 64 / 4 = 16 shards from the shuffled list of shards to each worker when you start iterating
+ >>> for example in ids:
+ ... pass
+ ```
+
+ In a distributed setup like PyTorch DDP with a PyTorch DataLoader and shuffling
+ ```python
+ >>> from datasets.distributed import split_dataset_by_node
+ >>> ids = ds.to_iterable_dataset(num_shards=512)
+ >>> ids = ids.shuffle(buffer_size=10_000, seed=42) # will shuffle the shards order and use a shuffle buffer when you start iterating
+ >>> ids = split_dataset_by_node(ds, world_size=8, rank=0) # will keep only 512 / 8 = 64 shards from the shuffled lists of shards when you start iterating
+ >>> dataloader = torch.utils.data.DataLoader(ids, num_workers=4) # will assign 64 / 4 = 16 shards from this node's list of shards to each worker when you start iterating
+ >>> for example in ids:
+ ... pass
+ ```
+
+ With shuffling and multiple epochs:
+ ```python
+ >>> ids = ds.to_iterable_dataset(num_shards=64)
+ >>> ids = ids.shuffle(buffer_size=10_000, seed=42) # will shuffle the shards order and use a shuffle buffer when you start iterating
+ >>> for epoch in range(n_epochs):
+ ... ids.set_epoch(epoch) # will use effective_seed = seed + epoch to shuffle the shards and for the shuffle buffer when you start iterating
+ ... for example in ids:
+ ... pass
+ ```
+ Feel free to also use [`IterableDataset.set_epoch`] when using a PyTorch DataLoader or in distributed setups.
+ """
+ from .iterable_dataset import ArrowExamplesIterable, IterableDataset
+
+ if self._format_type is not None:
+ if self._format_kwargs or (
+ self._format_columns is not None and set(self._format_columns) != set(self.column_names)
+ ):
+ raise NotImplementedError(
+ "Converting a formatted dataset with kwargs or selected columns to a formatted iterable dataset is not implemented yet. Please run `my_dataset = my_dataset.with_format(None)` before calling to_iterable_dataset"
+ )
+ if num_shards > len(self):
+ raise ValueError(
+ f"Unable to shard a dataset of size {len(self)} into {num_shards} shards (the number of shards exceeds the number of samples)."
+ )
+ if self._indices is not None:
+ logger.info(
+ "Converting an Arrow dataset to iterable but it has an indices mapping that can make it slower. "
+ "You can use `ds = ds.flatten_indices()` to write your dataset in contiguous chunks of data and have optimal speed."
+ )
+ shards = (
+ [copy.deepcopy(self)]
+ if num_shards == 1
+ else [
+ self.shard(num_shards=num_shards, index=shard_idx, contiguous=True) for shard_idx in range(num_shards)
+ ]
+ )
+ ex_iterable = ArrowExamplesIterable(
+ Dataset._generate_tables_from_shards,
+ kwargs={"shards": shards, "batch_size": config.DEFAULT_MAX_BATCH_SIZE},
+ )
+ ds = IterableDataset(ex_iterable, info=DatasetInfo(features=self.features))
+ if self._format_type:
+ ds = ds.with_format(self._format_type)
+ return ds
+
+ def _push_parquet_shards_to_hub_single(
+ self,
+ job_id: int,
+ num_jobs: int,
+ resolved_output_path: HfFileSystemResolvedPath,
+ data_dir: str,
+ split: str,
+ token: Optional[str],
+ create_pr: Optional[bool],
+ num_shards: int,
+ embed_external_files: bool,
+ writer_batch_size: int,
+ ):
+ div = num_shards // num_jobs
+ mod = num_shards % num_jobs
+ start = div * job_id + min(job_id, mod)
+ end = start + div + (1 if job_id < mod else 0)
+
+ index_shards = (
+ (start + i, self.shard(num_shards=end - start, index=i, contiguous=True)) for i in range(end - start)
+ )
+
+ api = HfApi(endpoint=config.HF_ENDPOINT, token=token, library_name="datasets", library_version=__version__)
+
+ additions: list[CommitOperationAdd] = []
+ new_parquet_paths: list[str] = []
+ uploaded_size = 0
+ for index, shard in index_shards:
+ if embed_external_files:
+ format = shard.format
+ shard = shard.with_format("arrow")
+ shard = shard.map(
+ embed_table_storage,
+ batched=True,
+ batch_size=writer_batch_size,
+ keep_in_memory=True,
+ )
+ shard = shard.with_format(**format)
+ shard_path_in_repo = f"{data_dir}/{split}-{index:05d}-of-{num_shards:05d}.parquet"
+ tmp_file = tempfile.NamedTemporaryFile(suffix=".parquet", delete=False)
+ try:
+ shard.to_parquet(tmp_file, batch_size=writer_batch_size)
+ uploaded_size += tmp_file.tell()
+ tmp_file.close()
+ new_parquet_paths.append(shard_path_in_repo)
+ if (
+ isinstance(resolved_output_path, HfFileSystemResolvedRepositoryPath)
+ and not resolved_output_path.path_in_repo
+ ):
+ shard_addition = CommitOperationAdd(path_in_repo=shard_path_in_repo, path_or_fileobj=tmp_file.name)
+ api.preupload_lfs_files(
+ repo_id=resolved_output_path.repo_id,
+ additions=[shard_addition],
+ repo_type=resolved_output_path.repo_type,
+ revision=resolved_output_path.revision,
+ create_pr=create_pr,
+ )
+ additions.append(shard_addition)
+ elif isinstance(resolved_output_path, HfFileSystemResolvedBucketPath):
+ if resolved_output_path.path:
+ shard_path_in_repo = resolved_output_path.path + "/" + shard_path_in_repo
+ api.batch_bucket_files(
+ bucket_id=resolved_output_path.bucket_id, add=[(tmp_file.name, shard_path_in_repo)]
+ )
+ else:
+ raise NotImplementedError(f"Bad HF path: {resolved_output_path}")
+ except (Exception, KeyboardInterrupt):
+ tmp_file.close()
+ Path(tmp_file.name).unlink()
+ raise
+ tmp_file.close()
+ Path(tmp_file.name).unlink()
+ yield job_id, False, 1
+
+ yield job_id, True, (additions, new_parquet_paths, uploaded_size)
+
+ def _push_parquet_shards_to_hub(
+ self,
+ resolved_output_path: HfFileSystemResolvedPath,
+ data_dir: str,
+ split: str,
+ token: Optional[str],
+ create_pr: Optional[bool],
+ max_shard_size: Optional[Union[int, str]],
+ num_shards: Optional[int],
+ embed_external_files: bool,
+ num_proc: Optional[int],
+ ) -> tuple[list[CommitOperationAdd], list[str], int, SplitInfo]:
+ """Pushes the dataset shards as Parquet files to the hub.
+
+ Returns:
+ additions (`List[CommitOperation]`): list of the `CommitOperationAdd` of the uploaded shards
+ new_parquet_paths (`List[str]`): list of paths of the new files uploaded to the output path,
+ relative to output path
+ features (`features`): features of the uploaded dataset
+ split_info (`int`): info of the uploaded split, including the approximate size in bytes of
+ the uploaded dataset after uncompression
+ uploaded_size (`int`): number of uploaded bytes to the repository or bucket
+ """
+ from .arrow_writer import get_writer_batch_size_from_data_size, get_writer_batch_size_from_features
+
+ dataset_nbytes = self._estimate_nbytes()
+ writer_batch_size = get_writer_batch_size_from_features(self.features) or get_writer_batch_size_from_data_size(
+ len(self), dataset_nbytes
+ )
+
+ # Find decodable columns, because if there are any, we need to:
+ # embed the bytes from the files in the shards
+ decodable_columns = (
+ [k for k, v in self._info.features.items() if require_decoding(v, ignore_decode_attribute=True)]
+ if embed_external_files
+ else []
+ )
+ embed_external_files = embed_external_files and bool(decodable_columns)
+
+ if num_shards is None:
+ max_shard_size = convert_file_size_to_int(max_shard_size or config.MAX_SHARD_SIZE)
+ num_shards = int(dataset_nbytes / max_shard_size) + 1
+ num_shards = max(num_shards, num_proc or 1)
+
+ additions: list[CommitOperationAdd] = []
+ new_parquet_paths: list[str] = []
+ uploaded_size = 0
+
+ num_jobs = num_proc or 1
+ if num_shards <= 1:
+ logger.warning(
+ f"Setting num_proc from {num_jobs} back to 1 for the {split} split to disable multiprocessing as it only contains one shard."
+ )
+ num_jobs = 1
+ elif num_shards < num_jobs:
+ logger.warning(
+ f"Setting num_proc from {num_jobs} to {num_shards} for the {split} split as it only contains {num_shards} shards."
+ )
+ num_proc = num_shards
+ kwargs_iterable = [
+ {
+ "self": self.shard(num_shards=num_jobs, index=job_id, contiguous=True),
+ "job_id": job_id,
+ "num_jobs": num_jobs,
+ "resolved_output_path": resolved_output_path,
+ "data_dir": data_dir,
+ "split": split,
+ "token": token,
+ "create_pr": create_pr,
+ "num_shards": num_shards,
+ "embed_external_files": embed_external_files,
+ "writer_batch_size": writer_batch_size,
+ }
+ for job_id in range(num_jobs)
+ ]
+ desc = "Uploading the dataset shards"
+ desc += f" (num_proc={num_proc})" if num_proc is not None and num_proc >= 1 else ""
+ pbar = hf_tqdm(
+ unit=" shards",
+ total=num_shards,
+ desc=desc,
+ )
+ with (
+ contextlib.nullcontext()
+ if num_proc is None or num_proc < 1
+ else mp.get_context("spawn").Pool(num_proc) as pool
+ ):
+ update_stream = (
+ Dataset._push_parquet_shards_to_hub_single(**kwargs_iterable[0])
+ if pool is None
+ else iflatmap_unordered(
+ pool,
+ Dataset._push_parquet_shards_to_hub_single,
+ kwargs_iterable=kwargs_iterable,
+ )
+ )
+ for job_id, done, content in update_stream:
+ if not done:
+ pbar.update(content)
+ else:
+ job_additions, job_new_parquet_paths, job_uploaded_size = content
+ additions += job_additions
+ new_parquet_paths += job_new_parquet_paths
+ uploaded_size += job_uploaded_size
+
+ split_info = SplitInfo(name=split, num_bytes=dataset_nbytes, num_examples=len(self))
+ return additions, new_parquet_paths, self.features, split_info, uploaded_size
+
+ def push_to_hub(
+ self,
+ repo_id: str,
+ config_name: str = "default",
+ set_default: Optional[bool] = None,
+ split: Optional[str] = None,
+ data_dir: Optional[str] = None,
+ commit_message: Optional[str] = None,
+ commit_description: Optional[str] = None,
+ private: Optional[bool] = None,
+ token: Optional[str] = None,
+ revision: Optional[str] = None,
+ create_pr: Optional[bool] = False,
+ max_shard_size: Optional[Union[int, str]] = None,
+ num_shards: Optional[int] = None,
+ embed_external_files: bool = True,
+ num_proc: Optional[int] = None,
+ ) -> CommitInfo:
+ """Pushes the dataset to the hub as a Parquet dataset.
+ The dataset is pushed using HTTP requests and does not need to have neither git or git-lfs installed.
+
+ The resulting Parquet files are self-contained by default. If your dataset contains [`Image`], [`Audio`] or [`Video`]
+ data, the Parquet files will store the bytes of your images or audio files.
+ You can disable this by setting `embed_external_files` to `False`.
+
+ Args:
+ repo_id (`str`):
+ The ID of the repository to push to in the following format: `/` or
+ `/`. Also accepts ``, which will default to the namespace
+ of the logged-in user.
+
+ It could also be a location inside a bucket, e.g. `buckets///...`
+ config_name (`str`, defaults to "default"):
+ The configuration name (or subset) of a dataset. Defaults to "default".
+ set_default (`bool`, *optional*):
+ Whether to set this configuration as the default one. Otherwise, the default configuration is the one
+ named "default".
+ split (`str`, *optional*):
+ The name of the split that will be given to that dataset. Defaults to `self.split`.
+ data_dir (`str`, *optional*):
+ Directory name that will contain the uploaded data files. Defaults to the `config_name` if different
+ from "default", else "data".
+
+
+ commit_message (`str`, *optional*):
+ Message to commit while pushing. Will default to `"Upload dataset"`.
+ commit_description (`str`, *optional*):
+ Description of the commit that will be created.
+ Additionally, description of the PR if a PR is created (`create_pr` is True).
+
+
+ private (`bool`, *optional*):
+ Whether to make the repo private. If `None` (default), the repo will be public unless the
+ organization's default is private. This value is ignored if the repo already exists.
+ token (`str`, *optional*):
+ An optional authentication token for the Hugging Face Hub. If no token is passed, will default
+ to the token saved locally when logging in with `huggingface-cli login`. Will raise an error
+ if no token is passed and the user is not logged-in.
+ revision (`str`, *optional*):
+ Branch to push the uploaded files to. Defaults to the `"main"` branch.
+
+
+ create_pr (`bool`, *optional*, defaults to `False`):
+ Whether to create a PR with the uploaded files or directly commit.
+
+
+ max_shard_size (`int` or `str`, *optional*, defaults to `"500MB"`):
+ The maximum size of the dataset shards to be uploaded to the hub. If expressed as a string, needs to be digits followed by
+ a unit (like `"5MB"`).
+ num_shards (`int`, *optional*):
+ Number of shards to write. By default, the number of shards depends on `max_shard_size`.
+
+
+ embed_external_files (`bool`, defaults to `True`):
+ Whether to embed file bytes in the shards.
+ In particular, this will do the following before the push for the fields of type:
+
+ - [`Audio`] and [`Image`]: remove local path information and embed file content in the Parquet files.
+ num_proc (`int`, *optional*, defaults to `None`):
+ Number of processes when preparing and uploading the dataset.
+ This is helpful if the dataset is made of many samples or media files to embed.
+ I uses "spawn" context to work with hf_xet, the rust client for fast uploads to HF.
+ Multiprocessing is disabled by default.
+
+
+
+ Return:
+ huggingface_hub.CommitInfo
+
+ Example:
+
+ ```python
+ >>> dataset.push_to_hub("/")
+ >>> dataset_dict.push_to_hub("/", private=True)
+ >>> dataset.push_to_hub("/", max_shard_size="1GB")
+ >>> dataset.push_to_hub("/", num_shards=1024)
+ ```
+
+ If your dataset has multiple splits (e.g. train/validation/test):
+
+ ```python
+ >>> train_dataset.push_to_hub("/", split="train")
+ >>> val_dataset.push_to_hub("/", split="validation")
+ >>> # later
+ >>> dataset = load_dataset("/")
+ >>> train_dataset = dataset["train"]
+ >>> val_dataset = dataset["validation"]
+ ```
+
+ If you want to add a new configuration (or subset) to a dataset (e.g. if the dataset has multiple tasks/versions/languages):
+
+ ```python
+ >>> english_dataset.push_to_hub("/", "en")
+ >>> french_dataset.push_to_hub("/", "fr")
+ >>> # later
+ >>> english_dataset = load_dataset("/", "en")
+ >>> french_dataset = load_dataset("/", "fr")
+ ```
+ """
+ if config_name == "data":
+ raise ValueError("`config_name` cannot be 'data'. Please, choose another name for configuration.")
+
+ if max_shard_size is not None and num_shards is not None:
+ raise ValueError(
+ "Failed to push_to_hub: please specify either max_shard_size or num_shards, but not both."
+ )
+
+ if split is None:
+ split = str(self.split) if self.split is not None else "train"
+
+ if not re.match(_split_re, split):
+ raise ValueError(f"Split name should match '{_split_re}' but got '{split}'.")
+
+ if not data_dir:
+ data_dir = config_name if config_name != "default" else "data" # for backward compatibility
+
+ api = HfApi(endpoint=config.HF_ENDPOINT, token=token, library_name="datasets", library_version=__version__)
+ if repo_id.startswith("buckets/"):
+ if BucketNotFoundError is None:
+ raise ImportError("Pushing datasets to buckets requires huggingface_hub>=1.6.0")
+ _, _namespace, _bucket_name, *_path_segments = repo_id.split("/")
+ try:
+ bucket_id = api.bucket_info(_namespace + "/" + _bucket_name).id
+ except BucketNotFoundError:
+ bucket_url = api.create_bucket(_namespace + "/" + _bucket_name, private=private, exist_ok=True)
+ bucket_id = bucket_url.bucket_id
+ path = "/".join(s for s in _path_segments if s)
+ return _push_to_bucket(
+ self,
+ bucket_id=bucket_id,
+ path=path,
+ config_name=config_name,
+ set_default=set_default,
+ split=split,
+ data_dir=data_dir,
+ token=token,
+ max_shard_size=max_shard_size,
+ num_shards=num_shards,
+ embed_external_files=embed_external_files,
+ num_proc=num_proc,
+ )
+ else:
+ try:
+ repo_id = api.repo_info(repo_id, repo_type="dataset").id
+ except RepositoryNotFoundError:
+ repo_url = api.create_repo(
+ repo_id,
+ repo_type="dataset",
+ private=private,
+ exist_ok=True,
+ )
+ repo_id = repo_url.repo_id
+
+ if revision is not None and not revision.startswith("refs/pr/"):
+ # We do not call create_branch for a PR reference: 400 Bad Request
+ api.create_branch(repo_id, branch=revision, repo_type="dataset", exist_ok=True)
+ return _push_to_repo(
+ self,
+ repo_id=repo_id,
+ config_name=config_name,
+ set_default=set_default,
+ split=split,
+ data_dir=data_dir,
+ commit_message=commit_message,
+ commit_description=commit_description,
+ token=token,
+ revision=revision,
+ create_pr=create_pr,
+ max_shard_size=max_shard_size,
+ num_shards=num_shards,
+ embed_external_files=embed_external_files,
+ num_proc=num_proc,
+ )
+
+ @transmit_format
+ @fingerprint_transform(inplace=False)
+ def add_column(
+ self,
+ name: str,
+ column: Union[list, np.ndarray],
+ new_fingerprint: Optional[str] = None,
+ feature: Optional[FeatureType] = None,
+ ):
+ """Add column to Dataset.
+
+
+
+ Args:
+ name (`str`):
+ Column name.
+ column (`list` or `np.array`):
+ Column data to be added.
+ feature (`FeatureType` or `None`, defaults to `None`):
+ Column datatype.
+
+ Returns:
+ [`Dataset`]
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> more_text = ds["text"]
+ >>> ds = ds.add_column(name="text_2", column=more_text)
+ >>> ds
+ Dataset({
+ features: ['text', 'label', 'text_2'],
+ num_rows: 1066
+ })
+ ```
+ """
+
+ if feature:
+ pyarrow_schema = Features({name: feature}).arrow_schema
+ else:
+ pyarrow_schema = None
+
+ column_table = InMemoryTable.from_pydict({name: column}, schema=pyarrow_schema)
+ _check_column_names(self._data.column_names + column_table.column_names)
+ dataset = self.flatten_indices() if self._indices is not None else self
+ # Concatenate tables horizontally
+ table = concat_tables([dataset._data, column_table], axis=1)
+ # Update features
+ info = dataset.info.copy()
+ info.features.update(Features.from_arrow_schema(column_table.schema))
+ table = update_metadata_with_features(table, info.features)
+ return Dataset(table, info=info, split=self.split, indices_table=None, fingerprint=new_fingerprint)
+
+ def add_faiss_index(
+ self,
+ column: str,
+ index_name: Optional[str] = None,
+ device: Optional[int] = None,
+ string_factory: Optional[str] = None,
+ metric_type: Optional[int] = None,
+ custom_index: Optional["faiss.Index"] = None, # noqa: F821
+ batch_size: int = 1000,
+ train_size: Optional[int] = None,
+ faiss_verbose: bool = False,
+ dtype=np.float32,
+ ):
+ """Add a dense index using Faiss for fast retrieval.
+ By default the index is done over the vectors of the specified column.
+ You can specify `device` if you want to run it on GPU (`device` must be the GPU index).
+ You can find more information about Faiss here:
+
+ - For [string factory](https://github.com/facebookresearch/faiss/wiki/The-index-factory)
+
+ Args:
+ column (`str`):
+ The column of the vectors to add to the index.
+ index_name (`str`, *optional*):
+ The `index_name`/identifier of the index.
+ This is the `index_name` that is used to call [`~datasets.Dataset.get_nearest_examples`] or [`~datasets.Dataset.search`].
+ By default it corresponds to `column`.
+ device (`Union[int, List[int]]`, *optional*):
+ If positive integer, this is the index of the GPU to use. If negative integer, use all GPUs.
+ If a list of positive integers is passed in, run only on those GPUs. By default it uses the CPU.
+ string_factory (`str`, *optional*):
+ This is passed to the index factory of Faiss to create the index.
+ Default index class is `IndexFlat`.
+ metric_type (`int`, *optional*):
+ Type of metric. Ex: `faiss.METRIC_INNER_PRODUCT` or `faiss.METRIC_L2`.
+ custom_index (`faiss.Index`, *optional*):
+ Custom Faiss index that you already have instantiated and configured for your needs.
+ batch_size (`int`):
+ Size of the batch to use while adding vectors to the `FaissIndex`. Default value is `1000`.
+
+ train_size (`int`, *optional*):
+ If the index needs a training step, specifies how many vectors will be used to train the index.
+ faiss_verbose (`bool`, defaults to `False`):
+ Enable the verbosity of the Faiss index.
+ dtype (`data-type`):
+ The dtype of the numpy arrays that are indexed.
+ Default is `np.float32`.
+
+ Example:
+
+ ```python
+ >>> ds = datasets.load_dataset('community-datasets/crime_and_punish', split='train')
+ >>> ds_with_embeddings = ds.map(lambda example: {'embeddings': embed(example['line']}))
+ >>> ds_with_embeddings.add_faiss_index(column='embeddings')
+ >>> # query
+ >>> scores, retrieved_examples = ds_with_embeddings.get_nearest_examples('embeddings', embed('my new query'), k=10)
+ >>> # save index
+ >>> ds_with_embeddings.save_faiss_index('embeddings', 'my_index.faiss')
+
+ >>> ds = datasets.load_dataset('community-datasets/crime_and_punish', split='train')
+ >>> # load index
+ >>> ds.load_faiss_index('embeddings', 'my_index.faiss')
+ >>> # query
+ >>> scores, retrieved_examples = ds.get_nearest_examples('embeddings', embed('my new query'), k=10)
+ ```
+ """
+ with self.formatted_as(type="numpy", columns=[column], dtype=dtype):
+ super().add_faiss_index(
+ column=column,
+ index_name=index_name,
+ device=device,
+ string_factory=string_factory,
+ metric_type=metric_type,
+ custom_index=custom_index,
+ batch_size=batch_size,
+ train_size=train_size,
+ faiss_verbose=faiss_verbose,
+ )
+ return self
+
+ def add_faiss_index_from_external_arrays(
+ self,
+ external_arrays: np.array,
+ index_name: str,
+ device: Optional[int] = None,
+ string_factory: Optional[str] = None,
+ metric_type: Optional[int] = None,
+ custom_index: Optional["faiss.Index"] = None, # noqa: F821
+ batch_size: int = 1000,
+ train_size: Optional[int] = None,
+ faiss_verbose: bool = False,
+ dtype=np.float32,
+ ):
+ """Add a dense index using Faiss for fast retrieval.
+ The index is created using the vectors of `external_arrays`.
+ You can specify `device` if you want to run it on GPU (`device` must be the GPU index).
+ You can find more information about Faiss here:
+
+ - For [string factory](https://github.com/facebookresearch/faiss/wiki/The-index-factory)
+
+ Args:
+ external_arrays (`np.array`):
+ If you want to use arrays from outside the lib for the index, you can set `external_arrays`.
+ It will use `external_arrays` to create the Faiss index instead of the arrays in the given `column`.
+ index_name (`str`):
+ The `index_name`/identifier of the index.
+ This is the `index_name` that is used to call [`~datasets.Dataset.get_nearest_examples`] or [`~datasets.Dataset.search`].
+ device (Optional `Union[int, List[int]]`, *optional*):
+ If positive integer, this is the index of the GPU to use. If negative integer, use all GPUs.
+ If a list of positive integers is passed in, run only on those GPUs. By default it uses the CPU.
+ string_factory (`str`, *optional*):
+ This is passed to the index factory of Faiss to create the index.
+ Default index class is `IndexFlat`.
+ metric_type (`int`, *optional*):
+ Type of metric. Ex: `faiss.faiss.METRIC_INNER_PRODUCT` or `faiss.METRIC_L2`.
+ custom_index (`faiss.Index`, *optional*):
+ Custom Faiss index that you already have instantiated and configured for your needs.
+ batch_size (`int`, *optional*):
+ Size of the batch to use while adding vectors to the FaissIndex. Default value is 1000.
+
+ train_size (`int`, *optional*):
+ If the index needs a training step, specifies how many vectors will be used to train the index.
+ faiss_verbose (`bool`, defaults to False):
+ Enable the verbosity of the Faiss index.
+ dtype (`numpy.dtype`):
+ The dtype of the numpy arrays that are indexed. Default is np.float32.
+ """
+ super().add_faiss_index_from_external_arrays(
+ external_arrays=external_arrays.astype(dtype),
+ index_name=index_name,
+ device=device,
+ string_factory=string_factory,
+ metric_type=metric_type,
+ custom_index=custom_index,
+ batch_size=batch_size,
+ train_size=train_size,
+ faiss_verbose=faiss_verbose,
+ )
+
+ def add_elasticsearch_index(
+ self,
+ column: str,
+ index_name: Optional[str] = None,
+ host: Optional[str] = None,
+ port: Optional[int] = None,
+ es_client: Optional["elasticsearch.Elasticsearch"] = None, # noqa: F821
+ es_index_name: Optional[str] = None,
+ es_index_config: Optional[dict] = None,
+ ):
+ """Add a text index using ElasticSearch for fast retrieval. This is done in-place.
+
+ Args:
+ column (`str`):
+ The column of the documents to add to the index.
+ index_name (`str`, *optional*):
+ The `index_name`/identifier of the index.
+ This is the index name that is used to call [`~Dataset.get_nearest_examples`] or [`~Dataset.search`].
+ By default it corresponds to `column`.
+ host (`str`, *optional*, defaults to `localhost`):
+ Host of where ElasticSearch is running.
+ port (`str`, *optional*, defaults to `9200`):
+ Port of where ElasticSearch is running.
+ es_client (`elasticsearch.Elasticsearch`, *optional*):
+ The elasticsearch client used to create the index if host and port are `None`.
+ es_index_name (`str`, *optional*):
+ The elasticsearch index name used to create the index.
+ es_index_config (`dict`, *optional*):
+ The configuration of the elasticsearch index.
+ Default config is:
+ ```
+ {
+ "settings": {
+ "number_of_shards": 1,
+ "analysis": {"analyzer": {"stop_standard": {"type": "standard", " stopwords": "_english_"}}},
+ },
+ "mappings": {
+ "properties": {
+ "text": {
+ "type": "text",
+ "analyzer": "standard",
+ "similarity": "BM25"
+ },
+ }
+ },
+ }
+ ```
+ Example:
+
+ ```python
+ >>> es_client = elasticsearch.Elasticsearch()
+ >>> ds = datasets.load_dataset('community-datasets/crime_and_punish', split='train')
+ >>> ds.add_elasticsearch_index(column='line', es_client=es_client, es_index_name="my_es_index")
+ >>> scores, retrieved_examples = ds.get_nearest_examples('line', 'my new query', k=10)
+ ```
+ """
+ with self.formatted_as(type=None, columns=[column]):
+ super().add_elasticsearch_index(
+ column=column,
+ index_name=index_name,
+ host=host,
+ port=port,
+ es_client=es_client,
+ es_index_name=es_index_name,
+ es_index_config=es_index_config,
+ )
+ return self
+
+ @transmit_format
+ @fingerprint_transform(inplace=False)
+ def add_item(self, item: dict, new_fingerprint: Optional[str] = None):
+ """Add item to Dataset.
+
+
+
+ Args:
+ item (`dict`):
+ Item data to be added.
+
+ Returns:
+ [`Dataset`]
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> new_review = {'label': 0, 'text': 'this movie is the absolute worst thing I have ever seen'}
+ >>> ds = ds.add_item(new_review)
+ >>> ds[-1]
+ {'label': 0, 'text': 'this movie is the absolute worst thing I have ever seen'}
+ ```
+ """
+ item_table = InMemoryTable.from_pydict({k: [v] for k, v in item.items()})
+ # We don't call _check_if_features_can_be_aligned here so this cast is "unsafe"
+ dset_features, item_features = _align_features(
+ [self._info.features, Features.from_arrow_schema(item_table.schema)]
+ )
+ # Cast to align the schemas of the tables and concatenate the tables
+ table = concat_tables(
+ [
+ self._data.cast(dset_features.arrow_schema) if self._info.features != dset_features else self._data,
+ item_table.cast(item_features.arrow_schema),
+ ]
+ )
+ if self._indices is None:
+ indices_table = None
+ else:
+ item_indices_array = pa.array([len(self._data)], type=pa.uint64())
+ item_indices_table = InMemoryTable.from_arrays([item_indices_array], names=["indices"])
+ indices_table = concat_tables([self._indices, item_indices_table])
+ info = self.info.copy()
+ info.features.update(item_features)
+ table = update_metadata_with_features(table, info.features)
+ return Dataset(
+ table,
+ info=info,
+ split=self.split,
+ indices_table=indices_table,
+ fingerprint=new_fingerprint,
+ )
+
+ def align_labels_with_mapping(self, label2id: dict, label_column: str) -> "Dataset":
+ """Align the dataset's label ID and label name mapping to match an input `label2id` mapping.
+ This is useful when you want to ensure that a model's predicted labels are aligned with the dataset.
+ The alignment in done using the lowercase label names.
+
+ Args:
+ label2id (`dict`):
+ The label name to ID mapping to align the dataset with.
+ label_column (`str`):
+ The column name of labels to align on.
+
+ Example:
+
+ ```python
+ >>> # dataset with mapping {'entailment': 0, 'neutral': 1, 'contradiction': 2}
+ >>> ds = load_dataset("nyu-mll/glue", "mnli", split="train")
+ >>> # mapping to align with
+ >>> label2id = {'CONTRADICTION': 0, 'NEUTRAL': 1, 'ENTAILMENT': 2}
+ >>> ds_aligned = ds.align_labels_with_mapping(label2id, "label")
+ ```
+
+ """
+ # Sanity checks
+ if label_column not in self._data.column_names:
+ raise ValueError(f"Column ({label_column}) not in table columns ({self._data.column_names}).")
+
+ label_feature = self._info.features[label_column]
+ if not (
+ isinstance(label_feature, ClassLabel)
+ or (isinstance(label_feature, Sequence) and isinstance(label_feature.feature, ClassLabel))
+ ):
+ raise ValueError(
+ f"Aligning labels with a mapping is only supported for {ClassLabel.__name__} column or {Sequence.__name__} column with the inner type {ClassLabel.__name__}, and column {label_feature} is of type {type(label_feature).__name__}."
+ )
+
+ # Sort input mapping by ID value to ensure the label names are aligned
+ label2id = dict(sorted(label2id.items(), key=lambda item: item[1]))
+ label_names = list(label2id.keys())
+ # Some label mappings use uppercase label names so we lowercase them during alignment
+ label2id = {k.lower(): v for k, v in label2id.items()}
+ int2str_function = (
+ label_feature.int2str if isinstance(label_feature, ClassLabel) else label_feature.feature.int2str
+ )
+
+ if isinstance(label_feature, ClassLabel):
+
+ def process_label_ids(batch):
+ dset_label_names = [
+ int2str_function(label_id).lower() if label_id is not None else None
+ for label_id in batch[label_column]
+ ]
+ batch[label_column] = [
+ label2id[label_name] if label_name is not None else None for label_name in dset_label_names
+ ]
+ return batch
+
+ else:
+
+ def process_label_ids(batch):
+ dset_label_names = [
+ [int2str_function(label_id).lower() if label_id is not None else None for label_id in seq]
+ for seq in batch[label_column]
+ ]
+ batch[label_column] = [
+ [label2id[label_name] if label_name is not None else None for label_name in seq]
+ for seq in dset_label_names
+ ]
+ return batch
+
+ features = self.features
+ features[label_column] = (
+ ClassLabel(num_classes=len(label_names), names=label_names)
+ if isinstance(label_feature, ClassLabel)
+ else List(ClassLabel(num_classes=len(label_names), names=label_names))
+ )
+ return self.map(process_label_ids, features=features, batched=True, desc="Aligning the labels")
+
+
+def _push_to_repo(
+ dset: Union["Dataset", "IterableDataset"],
+ repo_id: str,
+ config_name: str = "default",
+ set_default: Optional[bool] = None,
+ split: Optional[str] = None,
+ data_dir: Optional[str] = None,
+ commit_message: Optional[str] = None,
+ commit_description: Optional[str] = None,
+ token: Optional[str] = None,
+ revision: Optional[str] = None,
+ create_pr: Optional[bool] = False,
+ max_shard_size: Optional[Union[int, str]] = None,
+ num_shards: Optional[int] = None,
+ embed_external_files: bool = True,
+ num_proc: Optional[int] = None,
+) -> CommitInfo:
+ api = HfApi(endpoint=config.HF_ENDPOINT, token=token, library_name="datasets", library_version=__version__)
+ resolved_output_path = HfFileSystemResolvedRepositoryPath(
+ repo_id=repo_id, repo_type="dataset", revision=revision or "main", path_in_repo=""
+ )
+
+ additions, new_parquet_paths, features, split_info, uploaded_size = dset._push_parquet_shards_to_hub(
+ resolved_output_path=resolved_output_path,
+ data_dir=data_dir,
+ split=split,
+ token=token,
+ max_shard_size=max_shard_size,
+ num_shards=num_shards,
+ create_pr=create_pr,
+ embed_external_files=embed_external_files,
+ num_proc=num_proc,
+ )
+
+ commit_message = commit_message if commit_message is not None else "Upload dataset"
+ if len(additions) > config.UPLOADS_MAX_NUMBER_PER_COMMIT:
+ logger.info(
+ f"Number of files to upload is larger than {config.UPLOADS_MAX_NUMBER_PER_COMMIT}. Splitting the push into multiple commits."
+ )
+ num_commits = math.ceil(len(additions) / config.UPLOADS_MAX_NUMBER_PER_COMMIT)
+ for i in range(0, num_commits):
+ operations = additions[
+ i * config.UPLOADS_MAX_NUMBER_PER_COMMIT : (i + 1) * config.UPLOADS_MAX_NUMBER_PER_COMMIT
+ ]
+ for retry, sleep_time in enumerate(itertools.chain(range(10), itertools.repeat(30)), start=1):
+ # We need to retry if another commit happens at the same time
+ sleep_time *= 1 + random.random()
+ try:
+ commit_info = api.create_commit(
+ repo_id,
+ operations=operations,
+ commit_message=commit_message + f" (part {i:05d}-of-{num_commits:05d})",
+ commit_description=commit_description,
+ repo_type="dataset",
+ revision=revision,
+ create_pr=create_pr,
+ )
+ except HfHubHTTPError as err:
+ if (
+ err.__context__
+ and isinstance(err.__context__, HfHubHTTPError)
+ and err.__context__.response.status_code == 409
+ ):
+ # 409 is Conflict (another commit is in progress)
+ time.sleep(sleep_time)
+ logger.info(
+ f"Retrying intermediate commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})"
+ )
+ continue
+ else:
+ raise
+ break
+ logger.info(
+ f"Commit #{i + 1} completed"
+ + (f" (still {num_commits - i - 1} to go)" if num_commits - i - 1 else "")
+ + "."
+ )
+ last_commit_additions = []
+ else:
+ last_commit_additions = additions
+
+ for retry, sleep_time in enumerate(itertools.chain(range(10), itertools.repeat(30)), start=1):
+ # We need to retry if there was a commit in between in case it touched the dataset card data
+ sleep_time *= 1 + random.random()
+
+ # We make sure to get info from this commit
+ parent_commit = api.repo_info(repo_id, repo_type="dataset", revision=revision).sha
+ hf_path = HfFileSystemResolvedRepositoryPath(
+ repo_type=resolved_output_path.repo_type,
+ repo_id=resolved_output_path.repo_id,
+ revision=parent_commit,
+ path_in_repo=resolved_output_path.path_in_repo,
+ ).unresolve()
+ hffs = HfFileSystem(endpoint=config.HF_ENDPOINT, token=token)
+ dirfs = DirFileSystem(fs=hffs, path=hf_path)
+
+ # Check the files to delete
+ try:
+ files_to_delete = dirfs.glob(f"{data_dir}/{split}-*", detail=True)
+ except EntryNotFoundError: # needed for huggingface_hub<=1.7.1
+ files_to_delete = {}
+
+ # Don't delete the new files
+ deletions = [
+ CommitOperationDelete(path_in_repo=file_to_delete)
+ for file_to_delete in files_to_delete
+ if file_to_delete not in new_parquet_paths
+ ]
+ deleted_size = sum(file_info["size"] for file_info in files_to_delete.values())
+
+ # Update the dataset card
+ new_dataset_card, new_legacy_dataset_infos = _get_updated_dataset_card(
+ fs=dirfs,
+ config_name=config_name,
+ splits_info=[split_info],
+ features=features,
+ data_dir=data_dir,
+ set_default=set_default,
+ uploaded_sizes=[uploaded_size],
+ deleted_sizes=[deleted_size],
+ remove_other_splits=False,
+ )
+ dataset_card_additions = [
+ CommitOperationAdd(path_in_repo=config.REPOCARD_FILENAME, path_or_fileobj=str(new_dataset_card).encode())
+ ]
+ if new_legacy_dataset_infos:
+ dataset_card_additions.append(
+ CommitOperationAdd(
+ path_in_repo=config.DATASETDICT_INFOS_FILENAME,
+ path_or_fileobj=json.dumps(new_legacy_dataset_infos).encode("utf-8"),
+ )
+ )
+ operations = last_commit_additions + dataset_card_additions + deletions
+
+ try:
+ commit_info = api.create_commit(
+ repo_id,
+ operations=operations,
+ commit_message=commit_message,
+ commit_description=commit_description,
+ repo_type="dataset",
+ revision=revision,
+ create_pr=create_pr,
+ parent_commit=parent_commit,
+ )
+ except HfHubHTTPError as err:
+ if (
+ err.__context__
+ and isinstance(err.__context__, HfHubHTTPError)
+ and err.__context__.response.status_code in (412, 409)
+ ):
+ # 412 is Precondition failed (parent_commit isn't satisfied)
+ # 409 is Conflict (another commit is in progress)
+ time.sleep(sleep_time)
+ logger.info(
+ f"Retrying commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})"
+ )
+ continue
+ else:
+ raise
+ break
+
+ return commit_info
+
+
+def _push_to_bucket(
+ dset: Union["Dataset", "IterableDataset"],
+ bucket_id: str,
+ path: str,
+ config_name: str = "default",
+ set_default: Optional[bool] = None,
+ split: Optional[str] = None,
+ data_dir: Optional[str] = None,
+ token: Optional[str] = None,
+ create_pr: Optional[bool] = False,
+ max_shard_size: Optional[Union[int, str]] = None,
+ num_shards: Optional[int] = None,
+ embed_external_files: bool = True,
+ num_proc: Optional[int] = None,
+) -> None:
+ api = HfApi(endpoint=config.HF_ENDPOINT, token=token, library_name="datasets", library_version=__version__)
+ resolved_output_path = HfFileSystemResolvedBucketPath(bucket_id=bucket_id, path=path)
+ hf_path = resolved_output_path.unresolve()
+ hffs = HfFileSystem(endpoint=config.HF_ENDPOINT, token=token)
+ dirfs = DirFileSystem(fs=hffs, path=hf_path)
+
+ # Check the files to delete before uploading
+ try:
+ files_to_delete = dirfs.glob(f"{data_dir}/{split}-*", detail=True)
+ except EntryNotFoundError: # needed for huggingface_hub<=1.7.1
+ files_to_delete = {}
+
+ # Upload the Parquet files
+ _, new_parquet_paths, features, split_info, uploaded_size = dset._push_parquet_shards_to_hub(
+ resolved_output_path=resolved_output_path,
+ data_dir=data_dir,
+ split=split,
+ token=token,
+ max_shard_size=max_shard_size,
+ num_shards=num_shards,
+ create_pr=create_pr,
+ embed_external_files=embed_external_files,
+ num_proc=num_proc,
+ )
+
+ # Don't delete the new files
+ new_parquet_paths = set(new_parquet_paths)
+ delete = [file_to_delete for file_to_delete in files_to_delete if file_to_delete not in new_parquet_paths]
+ deleted_size = sum(file_info["size"] for file_info in files_to_delete.values())
+
+ # Update the dataset card
+ new_dataset_card, new_legacy_dataset_infos = _get_updated_dataset_card(
+ fs=dirfs,
+ config_name=config_name,
+ splits_info=[split_info],
+ features=features,
+ data_dir=data_dir,
+ set_default=set_default,
+ uploaded_sizes=[uploaded_size],
+ deleted_sizes=[deleted_size],
+ remove_other_splits=False,
+ )
+ path_prefix = (path + "/") if path else ""
+ add = [(str(new_dataset_card).encode(), path_prefix + config.REPOCARD_FILENAME)]
+ if new_legacy_dataset_infos:
+ add.append(
+ (json.dumps(new_legacy_dataset_infos).encode("utf-8"), path_prefix + config.DATASETDICT_INFOS_FILENAME)
+ )
+
+ # Upload dataset card and delete old files
+ api.batch_bucket_files(
+ bucket_id=bucket_id,
+ add=add,
+ delete=delete,
+ )
+
+
+def _get_updated_dataset_card(
+ fs: DirFileSystem,
+ config_name: str,
+ splits_info: list[SplitInfo],
+ features: Features,
+ data_dir: str,
+ set_default: Optional[bool],
+ uploaded_sizes: list[int],
+ deleted_sizes: list[int],
+ remove_other_splits: bool,
+) -> tuple[DatasetCard, Optional[dict]]:
+ """Update a dataset card in push_to_hub"""
+ # get the deprecated dataset_infos.json to update them
+ try:
+ legacy_dataset_info: dict = json.loads(fs.read_text(config.DATASETDICT_INFOS_FILENAME, encoding="utf-8")).get(
+ config_name, None
+ )
+ repo_info = DatasetInfo.from_dict(legacy_dataset_info) if legacy_dataset_info else None
+ except FileNotFoundError:
+ legacy_dataset_info = None
+ repo_info = None
+ # get the info from the README to update them
+ try:
+ dataset_card = DatasetCard(fs.read_text(config.REPOCARD_FILENAME, newline="", encoding="utf-8"))
+ dataset_card_data = dataset_card.data
+ metadata_configs = MetadataConfigs.from_dataset_card_data(dataset_card_data)
+ dataset_infos: DatasetInfosDict = DatasetInfosDict.from_dataset_card_data(dataset_card_data)
+ if dataset_infos and config_name in dataset_infos:
+ repo_info = dataset_infos[config_name]
+ else:
+ repo_info = None
+ except FileNotFoundError:
+ dataset_card = None
+ dataset_card_data = DatasetCardData()
+ metadata_configs = MetadataConfigs()
+ # update the total info to dump from existing info
+ if repo_info is not None and not remove_other_splits:
+ logger.info("Updating downloaded metadata with the new split" + ("s." if len(splits_info) > 1 else "."))
+ for split_info, deleted_size, uploaded_size in zip(splits_info, deleted_sizes, uploaded_sizes):
+ split = split_info.name
+ if repo_info.splits and any(s != split for s in repo_info.splits):
+ if features != repo_info.features:
+ raise ValueError(
+ f"Features of the new split don't match the features of the existing splits on the hub: {features} != {repo_info.features}"
+ )
+
+ if split in repo_info.splits:
+ repo_info.download_size -= deleted_size
+ repo_info.dataset_size -= repo_info.splits.get(split, SplitInfo()).num_bytes or 0
+
+ repo_info.download_checksums = None
+ repo_info.download_size = (repo_info.download_size or 0) + uploaded_size
+ repo_info.dataset_size = (repo_info.dataset_size or 0) + split_info.num_bytes
+ repo_info.size_in_bytes = repo_info.download_size + repo_info.dataset_size
+ repo_info.splits.pop(split, None)
+ repo_info.splits[split] = split_info
+ info_to_dump = repo_info
+ else:
+ info_to_dump = DatasetInfo(
+ config_name=config_name, features=features, splits=SplitDict(), download_size=0, dataset_size=0
+ )
+ for split_info, uploaded_size in zip(splits_info, uploaded_sizes):
+ info_to_dump.splits.add(split_info)
+ info_to_dump.download_size += uploaded_size
+ info_to_dump.dataset_size += split_info.num_bytes
+ info_to_dump.size_in_bytes = info_to_dump.download_size + info_to_dump.dataset_size
+ # create the metadata configs if it was uploaded with push_to_hub before metadata configs existed
+ repo_splits: list[str] = []
+ pattern = glob_pattern_to_regex(PUSH_TO_HUB_WITHOUT_METADATA_CONFIGS_SPLIT_PATTERN_SHARDED)
+ for file_path in fs.glob(PUSH_TO_HUB_WITHOUT_METADATA_CONFIGS_SPLIT_PATTERN_SHARDED.replace("{split}", "*")):
+ split_pattern_fields = string_to_dict(file_path, pattern)
+ assert split_pattern_fields is not None
+ repo_split = split_pattern_fields["split"]
+ if repo_split not in repo_splits:
+ repo_splits.append(repo_split)
+ if not metadata_configs and repo_splits:
+ default_metadata_configs_to_dump = {
+ "data_files": [{"split": split, "path": f"data/{split}-*"} for split in repo_splits]
+ }
+ MetadataConfigs({"default": default_metadata_configs_to_dump}).to_dataset_card_data(dataset_card_data)
+ # update the metadata configs
+ if config_name in metadata_configs:
+ metadata_config = metadata_configs[config_name]
+ if "data_files" in metadata_config:
+ data_files_to_dump = sanitize_patterns(metadata_config["data_files"])
+ else:
+ data_files_to_dump = {}
+ # add the new splits
+ for split_info in splits_info:
+ split = split_info.name
+ data_files_to_dump[split] = [f"{data_dir}/{split}-*"]
+ metadata_config_to_dump = {
+ "data_files": [
+ {
+ "split": _split,
+ "path": _pattern[0] if len(_pattern) == 1 else _pattern,
+ }
+ for _split, _pattern in data_files_to_dump.items()
+ ]
+ }
+ else:
+ metadata_config_to_dump = {
+ "data_files": [
+ {"split": split_info.name, "path": f"{data_dir}/{split_info.name}-*"} for split_info in splits_info
+ ]
+ }
+ configs_to_dump = {config_name: metadata_config_to_dump}
+ if set_default and config_name != "default":
+ if metadata_configs:
+ current_default_config_name = metadata_configs.get_default_config_name()
+ if current_default_config_name == "default":
+ raise ValueError(
+ "There exists a configuration named 'default'. To set a different configuration as default, "
+ "rename the 'default' one first."
+ )
+ if current_default_config_name:
+ _ = metadata_configs[current_default_config_name].pop("default")
+ configs_to_dump[current_default_config_name] = metadata_configs[current_default_config_name]
+ metadata_config_to_dump["default"] = True
+ # push to the deprecated dataset_infos.json
+ if legacy_dataset_info:
+ legacy_dataset_infos: dict = json.loads(fs.read_text(config.DATASETDICT_INFOS_FILENAME, encoding="utf-8"))
+ legacy_dataset_infos[config_name] = asdict(info_to_dump)
+ new_legacy_dataset_infos = json.dumps(dataset_infos, indent=4)
+ else:
+ new_legacy_dataset_infos = None
+ # push to README
+ DatasetInfosDict({config_name: info_to_dump}).to_dataset_card_data(dataset_card_data)
+ MetadataConfigs(configs_to_dump).to_dataset_card_data(dataset_card_data)
+ new_dataset_card = DatasetCard(f"---\n{dataset_card_data}\n---\n") if dataset_card is None else dataset_card
+ return new_dataset_card, new_legacy_dataset_infos
+
+
+def _concatenate_map_style_datasets(
+ dsets: list[Dataset],
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ axis: int = 0,
+):
+ """
+ Converts a list of :class:`Dataset` with the same schema into a single :class:`Dataset`.
+ When you concatenate on axis 0, missing data are filled with None values.
+
+ Args:
+ dsets (`List[datasets.Dataset]`): List of Datasets to concatenate.
+ info (:class:`DatasetInfo`, optional): Dataset information, like description, citation, etc.
+ split (:class:`NamedSplit`, optional): Name of the dataset split.
+ axis (``{0, 1}``, default ``0``, meaning over rows):
+ Axis to concatenate over, where ``0`` means over rows (vertically) and ``1`` means over columns
+ (horizontally).
+
+ *New in version 1.6.0*
+
+ Example:
+
+ ```py
+ >>> ds3 = _concatenate_map_style_datasets([ds1, ds2])
+ ```
+ """
+ # Ignore datasets with no rows
+ if any(dset.num_rows > 0 for dset in dsets):
+ dsets = [dset for dset in dsets if dset.num_rows > 0]
+ else:
+ # Return first dataset if all datasets are empty
+ return dsets[0]
+
+ # Perform checks (and a potential cast if axis=0)
+ if axis == 0:
+ _check_if_features_can_be_aligned([dset.features for dset in dsets])
+ else:
+ if not all(dset.num_rows == dsets[0].num_rows for dset in dsets):
+ raise ValueError("Number of rows must match for all datasets")
+ _check_column_names([col_name for dset in dsets for col_name in dset._data.column_names])
+
+ # Find common format or reset format
+ format = dsets[0].format
+ if any(dset.format != format for dset in dsets):
+ format = {}
+ logger.info("Some of the datasets have disparate format. Resetting the format of the concatenated dataset.")
+
+ def apply_offset_to_indices_table(table, offset):
+ if offset == 0:
+ return table
+ else:
+ array = table["indices"]
+ new_array = pc.add(array, pa.scalar(offset, type=pa.uint64()))
+ return InMemoryTable.from_arrays([new_array], names=["indices"])
+
+ # Concatenate indices if they exist
+ if any(dset._indices is not None for dset in dsets):
+ if axis == 0:
+ # Datasets with no indices tables are replaced with a dataset with an indices table in memory.
+ # Applying an offset to an indices table also brings the table in memory.
+ indices_tables = []
+ for i in range(len(dsets)):
+ if dsets[i]._indices is None:
+ dsets[i] = dsets[i]._select_with_indices_mapping(range(len(dsets[i])))
+ indices_tables.append(dsets[i]._indices)
+
+ # An offset needs to be applied to the indices before concatenating
+ offset = 0
+ for i in range(len(dsets)):
+ indices_tables[i] = apply_offset_to_indices_table(indices_tables[i], offset)
+ offset += len(dsets[i]._data)
+
+ # Concatenate indices
+ indices_tables = [t for t in indices_tables if len(t) > 0]
+ if indices_tables:
+ indices_table = concat_tables(indices_tables)
+ else:
+ indices_table = InMemoryTable.from_batches([], schema=pa.schema({"indices": pa.int64()}))
+ else:
+ if len(dsets) == 1:
+ indices_table = dsets[0]._indices
+ else:
+ for i in range(len(dsets)):
+ dsets[i] = dsets[i].flatten_indices()
+ indices_table = None
+ else:
+ indices_table = None
+
+ table = concat_tables([dset._data for dset in dsets], axis=axis)
+ if axis == 0:
+ features_list = _align_features([dset.features for dset in dsets])
+ else:
+ features_list = [dset.features for dset in dsets]
+ table = update_metadata_with_features(table, {k: v for features in features_list for k, v in features.items()})
+
+ # Concatenate infos
+ if info is None:
+ info = DatasetInfo.from_merge([dset.info for dset in dsets])
+ fingerprint = update_fingerprint(
+ "".join(dset._fingerprint for dset in dsets), _concatenate_map_style_datasets, {"info": info, "split": split}
+ )
+
+ # Make final concatenated dataset
+ concatenated_dataset = Dataset(
+ table,
+ info=info,
+ split=split,
+ indices_table=indices_table,
+ fingerprint=fingerprint,
+ )
+ concatenated_dataset.set_format(**format)
+ return concatenated_dataset
+
+
+def _interleave_map_style_datasets(
+ datasets: list["Dataset"],
+ probabilities: Optional[list[float]] = None,
+ seed: Optional[int] = None,
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ stopping_strategy: Literal[
+ "first_exhausted", "all_exhausted", "all_exhausted_without_replacement"
+ ] = "first_exhausted",
+ **kwargs,
+) -> "Dataset":
+ """
+ Interleave several map-style datasets (sources) into a single map-style dataset.
+ The new dataset is constructed by alternating between the sources to get the examples.
+ If `probabilities = None` (default) the new dataset is constructed by cycling between each source to get the examples.
+ If `probabilities` is not `None, the new dataset is constructed by getting examples from a random source at a time according to the provided probabilities.
+
+ Args:
+ datasets (`List[Dataset]`): list of datasets to interleave
+ probabilities (`List[float]`, optional, default None): If specified, the new dataset is constructed by sampling
+ examples from one source at a time according to these probabilities.
+ seed (`int`, optional, default None): The random seed used to choose a source for each example.
+ info (:class:`DatasetInfo`, optional): Dataset information, like description, citation, etc.
+ split (:class:`NamedSplit`, optional): Name of the dataset split.
+ stopping_strategy (`str`, defaults to `first_exhausted`):
+ Two strategies are proposed right now.
+ By default, `first_exhausted` is an undersampling strategy, i.e the dataset construction is stopped as soon as one dataset has ran out of samples.
+ If the strategy is `all_exhausted`, we use an oversampling strategy, i.e the dataset construction is stopped as soon as every samples of every dataset has been added at least once.
+ When strategy is `all_exhausted_without_replacement` we make sure that each sample in each dataset is sampled only once.
+ Note that if the strategy is `all_exhausted`, the interleaved dataset size can get enormous:
+ - with no probabilities, the resulting dataset will have max_length_datasets*nb_dataset samples.
+ - with given probabilities, the resulting dataset will have more samples if some datasets have really low probability of visiting.
+ **kwargs (additional keyword arguments): Keyword arguments to be passed to :meth:`datasets.Datasets.select` when selecting the indices used to interleave the datasets.
+
+ Output:
+ :class:`datasets.Dataset`
+ """
+ if stopping_strategy not in ["first_exhausted", "all_exhausted", "all_exhausted_without_replacement"]:
+ raise ValueError(
+ f"{stopping_strategy} stopping strategy in `interleave_datasets` is not implemented yet with a list of {type(datasets[0])}"
+ )
+
+ # To interleave the datasets, we concatenate them and then we re-order the indices
+ concatenated_datasets = _concatenate_map_style_datasets(datasets, info=info, split=split)
+
+ # Let's now build the indices to pass to .select()
+ lengths = [len(dset) for dset in datasets]
+ offsets = np.cumsum([0] + lengths[:-1])
+
+ # if stopping_strategy is "first_exhausted", it is an undersampling situation whereas it is an oversampling situation if it is "all_exhausted"
+ oversampling = stopping_strategy == "all_exhausted"
+
+ if probabilities is None and stopping_strategy == "all_exhausted_without_replacement":
+ # Without replacement situation with cycling between each sources
+ # Example: If lengths of the datasets are [3, 4, 3]
+ # Then the resulting indices should be [0, 3, 7, 1, 4, 8, 2, 5, 9, 6]
+ # We cycle through datasets until all are exhausted, but skip exhausted datasets
+
+ # Reasoning behind the following operation: keeping the first indices of each dataset
+ # while offsetting in order to correspond to the right indices of the concatenated dataset
+ # and flattening to effectively interleave the datasets. Then we remove the exausted datasets
+ # and we continue with the following indices, until all datasets are exhausted
+ chunks_boundaries = [0] + sorted(set(lengths))
+ chunks = zip(chunks_boundaries[:-1], chunks_boundaries[1:])
+ indices_chunks = []
+ for start, end in chunks:
+ indices_chunks.append((np.array(offsets).reshape(1, -1) + np.arange(start, end).reshape(-1, 1)).flatten())
+ exhausted_indices = [i for i in range(len(lengths)) if lengths[i] == end]
+ lengths = np.delete(lengths, exhausted_indices).tolist()
+ offsets = np.delete(offsets, exhausted_indices)
+ indices = np.concatenate(indices_chunks).tolist()
+
+ elif probabilities is None and not oversampling:
+ # Undersampling situation with cycling between each sources
+ # Example:: If lengths of the datasets are [3, 4, 5]
+ # Then the resulting indices should be [0, 3, 7, 1, 4, 8, 2, 5, 9]
+ # Note that we only have 3 examples per dataset since the first dataset ran out of examples
+
+ # Reasoning behind the following operation: keeping the min_length first indices of each dataset
+ # while offsetting in order to correspond to the right indices of the concatenated dataset
+ # and flattening to effectively interleave the datasets
+ indices = (offsets.reshape(1, -1) + np.arange(min(lengths)).reshape(-1, 1)).flatten().tolist()
+ elif probabilities is None:
+ # Oversampling situation with cycling between each sources
+ # Then the resulting indices should be [0, 3, 7, 1, 4, 8, 2, 5, 9, 0, 6, 10, 1, 3, 11]
+ # Note that we have 5 examples per dataset with a rolling window since the longest dataset has 5 samples
+
+ # Reasoning behind the following operation: for each dataset indices (i.e column) repeat the indices to have max_length indices per dataset
+ # For example, if the max_length is 5 and the i-th dataset has 3 samples, the i-th column will be [0,1,2,0,1]
+ indices = np.mod(np.arange(max(lengths)).reshape(-1, 1), np.array(lengths).reshape(1, -1))
+
+ # We have to keep the indices to their respective dataset offsets and to flatten to effectively interleave the datasets
+ indices = (indices + offsets).flatten().tolist()
+
+ else:
+ # boolean array indicating if at index i if the dataset_i has been fully exhausted
+ is_exhausted = np.full(len(lengths), False)
+
+ # if undersampling ("first_exhausted"), we stop as soon as one dataset is exhausted
+ # if oversampling ("all_exhausted"), we stop as soons as every dataset is exhausted, i.e as soon as every samples of every dataset has been visited at least once
+ bool_strategy_func = (
+ np.all if (oversampling or stopping_strategy == "all_exhausted_without_replacement") else np.any
+ )
+
+ def iter_random_indices():
+ """Get an infinite iterator that randomly samples the index of the source to pick examples from."""
+ rng = np.random.default_rng(seed)
+ while True:
+ yield from (int(i) for i in rng.choice(len(datasets), size=1000, p=probabilities))
+
+ current_index = [0] * len(datasets)
+ indices = []
+ for source_idx in iter_random_indices():
+ # If no oversampling, we stop as soon as a dataset has ran out of examples (np.any)
+ # Otherwise, we stop as soon as every dataset has ran out of examples (np.all)
+ if bool_strategy_func(is_exhausted):
+ # the stopping condition was reached, let's stop
+ break
+
+ # let's add the example at the current index of the `source_idx`-th dataset
+ # For without replacement sampling we additionally need to make sure the current source is not exhausted to not oversample.
+ if stopping_strategy != "all_exhausted_without_replacement" or not is_exhausted[source_idx]:
+ indices.append(current_index[source_idx] + offsets[source_idx])
+ current_index[source_idx] += 1
+
+ # we've ran out of examples for the current dataset, let's update our boolean array and bring the current_index back to 0
+ if current_index[source_idx] >= lengths[source_idx]:
+ is_exhausted[source_idx] = True
+ # We don't want to reset the iterator when stopping strategy is without replacement.
+ if stopping_strategy != "all_exhausted_without_replacement":
+ current_index[source_idx] = 0
+
+ return concatenated_datasets.select(indices, **kwargs)
+
+
+def _split_by_node_map_style_dataset(dataset: Dataset, rank: int, world_size: int) -> Dataset:
+ """
+ Split a dataset for the node at rank `rank` in a pool of nodes of size `world_size`.
+ Each node is assigned a chunk of data, e.g. rank 0 is given the first chunk of the dataset.
+ To maximize data loading throughput, chunks are made of contiguous data on disk if possible.
+
+ Args:
+ dataset ([`Dataset`]):
+ The dataset to split by node.
+ rank (`int`):
+ Rank of the current node.
+ world_size (`int`):
+ Total number of nodes.
+
+ Returns:
+ [`Dataset`]: The dataset to be used on the node at rank `rank`.
+ """
+ return dataset.shard(num_shards=world_size, index=rank, contiguous=True)
+
+
+# This is outside Dataset.filter as it needs to be picklable for multiprocessing
+
+
+def get_indices_from_mask_function(
+ function: Callable,
+ batched: bool,
+ with_indices: bool,
+ with_rank: bool,
+ input_columns: Optional[Union[str, list[str]]],
+ indices_mapping: Optional[Table] = None,
+ *args,
+ **fn_kwargs,
+):
+ if batched:
+ # we extract indices and rank from args
+ *inputs, indices, rank = args
+ additional_args = ()
+ if with_indices:
+ additional_args += (indices,)
+ if with_rank:
+ additional_args += (rank,)
+ mask = function(*inputs, *additional_args, **fn_kwargs)
+ if isinstance(mask, (pa.Array, pa.ChunkedArray)):
+ mask = mask.to_pylist()
+ else:
+ # we get batched data (to return less data than input) but `function` only accepts one example
+ # therefore we need to call `function` on each example of the batch to get the mask
+ *inputs, indices, rank = args
+ mask = []
+ if input_columns is None:
+ # inputs only contains a batch of examples
+ batch: dict = inputs[0]
+ num_examples = len(batch[next(iter(batch.keys()))])
+ for i in range(num_examples):
+ example = {key: batch[key][i] for key in batch}
+ additional_args = ()
+ if with_indices:
+ additional_args += (indices[i],)
+ if with_rank:
+ additional_args += (rank,)
+ mask.append(function(example, *additional_args, **fn_kwargs))
+ else:
+ # inputs is a list of columns
+ columns: list[list] = inputs
+ num_examples = len(columns[0])
+ for i in range(num_examples):
+ input = [column[i] for column in columns]
+ additional_args = ()
+ if with_indices:
+ additional_args += (indices[i],)
+ if with_rank:
+ additional_args += (rank,)
+ mask.append(function(*input, *additional_args, **fn_kwargs))
+ indices_array = [i for i, to_keep in zip(indices, mask) if to_keep]
+ if indices_mapping is not None:
+ indices_array = pa.array(indices_array, type=pa.uint64())
+ indices_array = indices_mapping.column(0).take(indices_array)
+ indices_array = indices_array.to_pylist()
+ return {"indices": indices_array}
+
+
+async def async_get_indices_from_mask_function(
+ function: Callable,
+ batched: bool,
+ with_indices: bool,
+ with_rank: bool,
+ input_columns: Optional[Union[str, list[str]]],
+ indices_mapping: Optional[Table] = None,
+ *args,
+ **fn_kwargs,
+):
+ """same function but async"""
+ if batched:
+ # we extract indices and rank from args
+ *inputs, indices, rank = args
+ additional_args = ()
+ if with_indices:
+ additional_args += (indices,)
+ if with_rank:
+ additional_args += (rank,)
+ mask = await function(*inputs, *additional_args, **fn_kwargs)
+ if isinstance(mask, (pa.Array, pa.ChunkedArray)):
+ mask = mask.to_pylist()
+ else:
+ # we get batched data (to return less data than input) but `function` only accepts one example
+ # therefore we need to call `function` on each example of the batch to get the mask
+ *inputs, indices, rank = args
+ mask = []
+ if input_columns is None:
+ # inputs only contains a batch of examples
+ batch: dict = inputs[0]
+ num_examples = len(batch[next(iter(batch.keys()))])
+ for i in range(num_examples):
+ example = {key: batch[key][i] for key in batch}
+ additional_args = ()
+ if with_indices:
+ additional_args += (indices[i],)
+ if with_rank:
+ additional_args += (rank,)
+ mask.append(await function(example, *additional_args, **fn_kwargs))
+ else:
+ # inputs is a list of columns
+ columns: list[list] = inputs
+ num_examples = len(columns[0])
+ for i in range(num_examples):
+ input = [column[i] for column in columns]
+ additional_args = ()
+ if with_indices:
+ additional_args += (indices[i],)
+ if with_rank:
+ additional_args += (rank,)
+ mask.append(await function(*input, *additional_args, **fn_kwargs))
+ indices_array = [i for i, to_keep in zip(indices, mask) if to_keep]
+ if indices_mapping is not None:
+ indices_array = pa.array(indices_array, type=pa.uint64())
+ indices_array = indices_mapping.column(0).take(indices_array)
+ indices_array = indices_array.to_pylist()
+ return {"indices": indices_array}
diff --git a/venv/Lib/site-packages/datasets/arrow_reader.py b/venv/Lib/site-packages/datasets/arrow_reader.py
new file mode 100644
index 0000000000000000000000000000000000000000..d9cf2cf0f4bd239a8f67dd76a279a4e81c26d0d6
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/arrow_reader.py
@@ -0,0 +1,620 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Arrow ArrowReader."""
+
+import copy
+import math
+import os
+import re
+from dataclasses import dataclass
+from functools import partial
+from typing import TYPE_CHECKING, Optional, Union
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+from tqdm.contrib.concurrent import thread_map
+
+from .download.download_config import DownloadConfig # noqa: F401
+from .naming import _split_re, filenames_for_dataset_split
+from .table import InMemoryTable, MemoryMappedTable, Table, concat_tables
+from .utils import logging
+from .utils import tqdm as hf_tqdm
+
+
+if TYPE_CHECKING:
+ from .info import DatasetInfo # noqa: F401
+ from .splits import Split, SplitInfo # noqa: F401
+
+
+logger = logging.get_logger(__name__)
+
+HF_GCP_BASE_URL = "https://storage.googleapis.com/huggingface-nlp/cache/datasets"
+
+_SUB_SPEC_RE = re.compile(
+ rf"""
+^
+ (?P{_split_re[1:-1]})
+ (\[
+ ((?P-?[\d_]+)
+ (?P%)?)?
+ :
+ ((?P-?[\d_]+)
+ (?P%)?)?
+ \])?(\((?P[^\)]*)\))?
+$
+""", # remove ^ and $
+ re.X,
+)
+
+_ADDITION_SEP_RE = re.compile(r"\s*\+\s*")
+
+
+class DatasetNotOnHfGcsError(ConnectionError):
+ """When you can't get the dataset from the Hf google cloud storage"""
+
+ pass
+
+
+class MissingFilesOnHfGcsError(ConnectionError):
+ """When some files are missing on the Hf oogle cloud storage"""
+
+ pass
+
+
+@dataclass(frozen=True)
+class FileInstructions:
+ """The file instructions associated with a split ReadInstruction.
+
+ Attributes:
+ num_examples: `int`, The total number of examples
+ file_instructions: List[dict(filename, skip, take)], the files information.
+ The filenames contains the relative path, not absolute.
+ skip/take indicates which example read in the file: `ds.slice(skip, take)`
+ """
+
+ num_examples: int
+ file_instructions: list[dict]
+
+
+def make_file_instructions(
+ name: str,
+ split_infos: list["SplitInfo"],
+ instruction: Union[str, "ReadInstruction"],
+ filetype_suffix: Optional[str] = None,
+ prefix_path: Optional[str] = None,
+) -> FileInstructions:
+ """Returns instructions of the split dict.
+
+ Args:
+ name (`str`): Name of the dataset.
+ split_infos (`list` of `[SplitInfo]`): Dataset splits information.
+ instruction ([`ReadInstruction`] or `str`): Reading instruction for a dataset.
+ filetype_suffix (`str`, *optional*): Suffix of dataset files, e.g. 'arrow' or 'parquet'.
+ prefix_path (`str`, *optional*): Prefix of dataset files, e.g. directory name.
+
+ Returns:
+ [`FileInstructions`]
+ """
+ if not isinstance(name, str):
+ raise TypeError(f"Expected str 'name', but got: {type(name).__name__}")
+ elif not name:
+ raise ValueError("Expected non-empty str 'name'")
+ name2len = {info.name: info.num_examples for info in split_infos}
+ name2shard_lengths = {info.name: info.shard_lengths for info in split_infos}
+ name2filenames = {
+ info.name: filenames_for_dataset_split(
+ path=prefix_path,
+ dataset_name=name,
+ split=info.name,
+ filetype_suffix=filetype_suffix,
+ shard_lengths=name2shard_lengths[info.name],
+ )
+ for info in split_infos
+ }
+ if not isinstance(instruction, ReadInstruction):
+ instruction = ReadInstruction.from_spec(instruction)
+ # Create the absolute instruction (per split)
+ absolute_instructions = instruction.to_absolute(name2len)
+
+ # For each split, return the files instruction (skip/take)
+ file_instructions = []
+ num_examples = 0
+ for abs_instr in absolute_instructions:
+ split_length = name2len[abs_instr.splitname]
+ filenames = name2filenames[abs_instr.splitname]
+ shard_lengths = name2shard_lengths[abs_instr.splitname]
+ from_ = 0 if abs_instr.from_ is None else abs_instr.from_
+ to = split_length if abs_instr.to is None else abs_instr.to
+ if shard_lengths is None: # not sharded
+ for filename in filenames:
+ take = to - from_
+ if take == 0:
+ continue
+ num_examples += take
+ file_instructions.append({"filename": filename, "skip": from_, "take": take})
+ else: # sharded
+ index_start = 0 # Beginning (included) of moving window.
+ index_end = 0 # End (excluded) of moving window.
+ for filename, shard_length in zip(filenames, shard_lengths):
+ index_end += shard_length
+ if from_ < index_end and to > index_start: # There is something to take.
+ skip = from_ - index_start if from_ > index_start else 0
+ take = to - index_start - skip if to < index_end else -1
+ if take == 0:
+ continue
+ file_instructions.append({"filename": filename, "skip": skip, "take": take})
+ num_examples += shard_length - skip if take == -1 else take
+ index_start += shard_length
+ return FileInstructions(
+ num_examples=num_examples,
+ file_instructions=file_instructions,
+ )
+
+
+class BaseReader:
+ """
+ Build a Dataset object out of Instruction instance(s).
+ """
+
+ def __init__(self, path: str, info: Optional["DatasetInfo"]):
+ """Initializes ArrowReader.
+
+ Args:
+ path (str): path where tfrecords are stored.
+ info (DatasetInfo): info about the dataset.
+ """
+ self._path: str = path
+ self._info: Optional["DatasetInfo"] = info
+ self._filetype_suffix: Optional[str] = None
+
+ def _get_table_from_filename(self, filename_skip_take, in_memory=False) -> Table:
+ """Returns a Dataset instance from given (filename, skip, take)."""
+ raise NotImplementedError
+
+ def _read_files(self, files, in_memory=False) -> Table:
+ """Returns Dataset for given file instructions.
+
+ Args:
+ files: List[dict(filename, skip, take)], the files information.
+ The filenames contain the absolute path, not relative.
+ skip/take indicates which example read in the file: `ds.slice(skip, take)`
+ in_memory (bool, default False): Whether to copy the data in-memory.
+ """
+ if len(files) == 0 or not all(isinstance(f, dict) for f in files):
+ raise ValueError("please provide valid file informations")
+ files = copy.deepcopy(files)
+ for f in files:
+ f["filename"] = os.path.join(self._path, f["filename"])
+
+ pa_tables = thread_map(
+ partial(self._get_table_from_filename, in_memory=in_memory),
+ files,
+ tqdm_class=hf_tqdm,
+ desc="Loading dataset shards",
+ # set `disable=None` rather than `disable=False` by default to disable progress bar when no TTY attached
+ disable=len(files) <= 16 or None,
+ )
+ pa_tables = [t for t in pa_tables if len(t) > 0]
+ if not pa_tables and (self._info is None or self._info.features is None):
+ raise ValueError(
+ "Tried to read an empty table. Please specify at least info.features to create an empty table with the right type."
+ )
+ pa_tables = pa_tables or [InMemoryTable.from_batches([], schema=pa.schema(self._info.features.type))]
+ pa_table = concat_tables(pa_tables) if len(pa_tables) != 1 else pa_tables[0]
+ return pa_table
+
+ def get_file_instructions(self, name, instruction, split_infos):
+ """Return list of dict {'filename': str, 'skip': int, 'take': int}"""
+ file_instructions = make_file_instructions(
+ name, split_infos, instruction, filetype_suffix=self._filetype_suffix, prefix_path=self._path
+ )
+ files = file_instructions.file_instructions
+ return files
+
+ def read(
+ self,
+ name,
+ instructions,
+ split_infos,
+ in_memory=False,
+ ):
+ """Returns Dataset instance(s).
+
+ Args:
+ name (str): name of the dataset.
+ instructions (ReadInstruction): instructions to read.
+ Instruction can be string and will then be passed to the Instruction
+ constructor as it.
+ split_infos (list of SplitInfo proto): the available splits for dataset.
+ in_memory (bool, default False): Whether to copy the data in-memory.
+
+ Returns:
+ kwargs to build a single Dataset instance.
+ """
+
+ files = self.get_file_instructions(name, instructions, split_infos)
+ if not files:
+ msg = f'Instruction "{instructions}" corresponds to no data!'
+ raise ValueError(msg)
+ return self.read_files(files=files, original_instructions=instructions, in_memory=in_memory)
+
+ def read_files(
+ self,
+ files: list[dict],
+ original_instructions: Union[None, "ReadInstruction", "Split"] = None,
+ in_memory=False,
+ ):
+ """Returns single Dataset instance for the set of file instructions.
+
+ Args:
+ files: List[dict(filename, skip, take)], the files information.
+ The filenames contains the relative path, not absolute.
+ skip/take indicates which example read in the file: `ds.skip().take()`
+ original_instructions: store the original instructions used to build the dataset split in the dataset.
+ in_memory (bool, default False): Whether to copy the data in-memory.
+
+ Returns:
+ kwargs to build a Dataset instance.
+ """
+ # Prepend path to filename
+ pa_table = self._read_files(files, in_memory=in_memory)
+ # If original_instructions is not None, convert it to a human-readable NamedSplit
+ if original_instructions is not None:
+ from .splits import Split # noqa
+
+ split = Split(str(original_instructions))
+ else:
+ split = None
+ dataset_kwargs = {"arrow_table": pa_table, "info": self._info, "split": split}
+ return dataset_kwargs
+
+
+class ArrowReader(BaseReader):
+ """
+ Build a Dataset object out of Instruction instance(s).
+ This Reader uses either memory mapping or file descriptors (in-memory) on arrow files.
+ """
+
+ def __init__(self, path: str, info: Optional["DatasetInfo"]):
+ """Initializes ArrowReader.
+
+ Args:
+ path (str): path where Arrow files are stored.
+ info (DatasetInfo): info about the dataset.
+ """
+ super().__init__(path, info)
+ self._filetype_suffix = "arrow"
+
+ def _get_table_from_filename(self, filename_skip_take, in_memory=False) -> Table:
+ """Returns a Dataset instance from given (filename, skip, take)."""
+ filename, skip, take = (
+ filename_skip_take["filename"],
+ filename_skip_take["skip"] if "skip" in filename_skip_take else None,
+ filename_skip_take["take"] if "take" in filename_skip_take else None,
+ )
+ table = ArrowReader.read_table(filename, in_memory=in_memory)
+ if take == -1:
+ take = len(table) - skip
+ # here we don't want to slice an empty table, or it may segfault
+ if skip is not None and take is not None and not (skip == 0 and take == len(table)):
+ table = table.slice(skip, take)
+ return table
+
+ @staticmethod
+ def read_table(filename, in_memory=False) -> Table:
+ """
+ Read table from file.
+
+ Args:
+ filename (str): File name of the table.
+ in_memory (bool, default=False): Whether to copy the data in-memory.
+
+ Returns:
+ pyarrow.Table
+ """
+ table_cls = InMemoryTable if in_memory else MemoryMappedTable
+ return table_cls.from_file(filename)
+
+
+class ParquetReader(BaseReader):
+ """
+ Build a Dataset object out of Instruction instance(s).
+ This Reader uses memory mapping on parquet files.
+ """
+
+ def __init__(self, path: str, info: Optional["DatasetInfo"]):
+ """Initializes ParquetReader.
+
+ Args:
+ path (str): path where tfrecords are stored.
+ info (DatasetInfo): info about the dataset.
+ """
+ super().__init__(path, info)
+ self._filetype_suffix = "parquet"
+
+ def _get_table_from_filename(self, filename_skip_take, **kwargs):
+ """Returns a Dataset instance from given (filename, skip, take)."""
+ filename, skip, take = (
+ filename_skip_take["filename"],
+ filename_skip_take["skip"] if "skip" in filename_skip_take else None,
+ filename_skip_take["take"] if "take" in filename_skip_take else None,
+ )
+ # Parquet read_table always loads data in memory, independently of memory_map
+ pa_table = pq.read_table(filename, memory_map=True)
+ # here we don't want to slice an empty table, or it may segfault
+ if skip is not None and take is not None and not (skip == 0 and take == len(pa_table)):
+ pa_table = pa_table.slice(skip, take)
+ return pa_table
+
+
+@dataclass(frozen=True)
+class _AbsoluteInstruction:
+ """A machine friendly slice: defined absolute positive boundaries."""
+
+ splitname: str
+ from_: int # uint (starting index).
+ to: int # uint (ending index).
+
+
+@dataclass(frozen=True)
+class _RelativeInstruction:
+ """Represents a single parsed slicing instruction, can use % and negatives."""
+
+ splitname: str
+ from_: Optional[int] = None # int (starting index) or None if no lower boundary.
+ to: Optional[int] = None # int (ending index) or None if no upper boundary.
+ unit: Optional[str] = None
+ rounding: Optional[str] = None
+
+ def __post_init__(self):
+ if self.unit is not None and self.unit not in ["%", "abs"]:
+ raise ValueError("unit must be either % or abs")
+ if self.rounding is not None and self.rounding not in ["closest", "pct1_dropremainder"]:
+ raise ValueError("rounding must be either closest or pct1_dropremainder")
+ if self.unit != "%" and self.rounding is not None:
+ raise ValueError("It is forbidden to specify rounding if not using percent slicing.")
+ if self.unit == "%" and self.from_ is not None and abs(self.from_) > 100:
+ raise ValueError("Percent slice boundaries must be > -100 and < 100.")
+ if self.unit == "%" and self.to is not None and abs(self.to) > 100:
+ raise ValueError("Percent slice boundaries must be > -100 and < 100.")
+ # Update via __dict__ due to instance being "frozen"
+ self.__dict__["rounding"] = "closest" if self.rounding is None and self.unit == "%" else self.rounding
+
+
+def _str_to_read_instruction(spec):
+ """Returns ReadInstruction for given string."""
+ res = _SUB_SPEC_RE.match(spec)
+ if not res:
+ raise ValueError(f"Unrecognized instruction format: {spec}")
+ unit = "%" if res.group("from_pct") or res.group("to_pct") else "abs"
+ return ReadInstruction(
+ split_name=res.group("split"),
+ rounding=res.group("rounding"),
+ from_=int(res.group("from")) if res.group("from") else None,
+ to=int(res.group("to")) if res.group("to") else None,
+ unit=unit,
+ )
+
+
+def _pct_to_abs_pct1(boundary, num_examples):
+ # Using math.trunc here, since -99.5% should give -99%, not -100%.
+ if num_examples < 100:
+ msg = (
+ 'Using "pct1_dropremainder" rounding on a split with less than 100 '
+ "elements is forbidden: it always results in an empty dataset."
+ )
+ raise ValueError(msg)
+ return boundary * math.trunc(num_examples / 100.0)
+
+
+def _pct_to_abs_closest(boundary, num_examples):
+ return int(round(boundary * num_examples / 100.0))
+
+
+def _rel_to_abs_instr(rel_instr, name2len):
+ """Returns _AbsoluteInstruction instance for given RelativeInstruction.
+
+ Args:
+ rel_instr: RelativeInstruction instance.
+ name2len: dict {split_name: num_examples}.
+ """
+ pct_to_abs = _pct_to_abs_closest if rel_instr.rounding == "closest" else _pct_to_abs_pct1
+ split = rel_instr.splitname
+ if split not in name2len:
+ raise ValueError(f'Unknown split "{split}". Should be one of {list(name2len)}.')
+ num_examples = name2len[split]
+ from_ = rel_instr.from_
+ to = rel_instr.to
+ if rel_instr.unit == "%":
+ from_ = 0 if from_ is None else pct_to_abs(from_, num_examples)
+ to = num_examples if to is None else pct_to_abs(to, num_examples)
+ else:
+ from_ = 0 if from_ is None else from_
+ to = num_examples if to is None else to
+ if from_ < 0:
+ from_ = max(num_examples + from_, 0)
+ if to < 0:
+ to = max(num_examples + to, 0)
+ from_ = min(from_, num_examples)
+ to = min(to, num_examples)
+ return _AbsoluteInstruction(split, from_, to)
+
+
+class ReadInstruction:
+ """Reading instruction for a dataset.
+
+ Examples::
+
+ # The following lines are equivalent:
+ ds = datasets.load_dataset('ylecun/mnist', split='test[:33%]')
+ ds = datasets.load_dataset('ylecun/mnist', split=datasets.ReadInstruction.from_spec('test[:33%]'))
+ ds = datasets.load_dataset('ylecun/mnist', split=datasets.ReadInstruction('test', to=33, unit='%'))
+ ds = datasets.load_dataset('ylecun/mnist', split=datasets.ReadInstruction(
+ 'test', from_=0, to=33, unit='%'))
+
+ # The following lines are equivalent:
+ ds = datasets.load_dataset('ylecun/mnist', split='test[:33%]+train[1:-1]')
+ ds = datasets.load_dataset('ylecun/mnist', split=datasets.ReadInstruction.from_spec(
+ 'test[:33%]+train[1:-1]'))
+ ds = datasets.load_dataset('ylecun/mnist', split=(
+ datasets.ReadInstruction('test', to=33, unit='%') +
+ datasets.ReadInstruction('train', from_=1, to=-1, unit='abs')))
+
+ # The following lines are equivalent:
+ ds = datasets.load_dataset('ylecun/mnist', split='test[:33%](pct1_dropremainder)')
+ ds = datasets.load_dataset('ylecun/mnist', split=datasets.ReadInstruction.from_spec(
+ 'test[:33%](pct1_dropremainder)'))
+ ds = datasets.load_dataset('ylecun/mnist', split=datasets.ReadInstruction(
+ 'test', from_=0, to=33, unit='%', rounding="pct1_dropremainder"))
+
+ # 10-fold validation:
+ tests = datasets.load_dataset(
+ 'ylecun/mnist',
+ [datasets.ReadInstruction('train', from_=k, to=k+10, unit='%')
+ for k in range(0, 100, 10)])
+ trains = datasets.load_dataset(
+ 'ylecun/mnist',
+ [datasets.ReadInstruction('train', to=k, unit='%') + datasets.ReadInstruction('train', from_=k+10, unit='%')
+ for k in range(0, 100, 10)])
+
+ """
+
+ def _init(self, relative_instructions):
+ # Private initializer.
+ self._relative_instructions = relative_instructions
+
+ @classmethod
+ def _read_instruction_from_relative_instructions(cls, relative_instructions):
+ """Returns ReadInstruction obj initialized with relative_instructions."""
+ # Use __new__ to bypass __init__ used by public API and not conveniant here.
+ result = cls.__new__(cls)
+ result._init(relative_instructions) # pylint: disable=protected-access
+ return result
+
+ def __init__(self, split_name, rounding=None, from_=None, to=None, unit=None):
+ """Initialize ReadInstruction.
+
+ Args:
+ split_name (str): name of the split to read. Eg: 'train'.
+ rounding (str, optional): The rounding behaviour to use when percent slicing is
+ used. Ignored when slicing with absolute indices.
+ Possible values:
+ - 'closest' (default): The specified percentages are rounded to the
+ closest value. Use this if you want specified percents to be as
+ much exact as possible.
+ - 'pct1_dropremainder': the specified percentages are treated as
+ multiple of 1%. Use this option if you want consistency. Eg:
+ len(5%) == 5 * len(1%).
+ Using this option, one might not be able to use the full set of
+ examples, if the number of those is not a multiple of 100.
+ from_ (int):
+ to (int): alternative way of specifying slicing boundaries. If any of
+ {from_, to, unit} argument is used, slicing cannot be specified as
+ string.
+ unit (str): optional, one of:
+ '%': to set the slicing unit as percents of the split size.
+ 'abs': to set the slicing unit as absolute numbers.
+ """
+ # This constructor is not always called. See factory method
+ # `_read_instruction_from_relative_instructions`. Common init instructions
+ # MUST be placed in the _init method.
+ self._init([_RelativeInstruction(split_name, from_, to, unit, rounding)])
+
+ @classmethod
+ def from_spec(cls, spec):
+ """Creates a `ReadInstruction` instance out of a string spec.
+
+ Args:
+ spec (`str`):
+ Split(s) + optional slice(s) to read + optional rounding
+ if percents are used as the slicing unit. A slice can be specified,
+ using absolute numbers (`int`) or percentages (`int`).
+
+ Examples:
+
+ ```
+ test: test split.
+ test + validation: test split + validation split.
+ test[10:]: test split, minus its first 10 records.
+ test[:10%]: first 10% records of test split.
+ test[:20%](pct1_dropremainder): first 10% records, rounded with the pct1_dropremainder rounding.
+ test[:-5%]+train[40%:60%]: first 95% of test + middle 20% of train.
+ ```
+
+ Returns:
+ ReadInstruction instance.
+ """
+ spec = str(spec) # Need to convert to str in case of NamedSplit instance.
+ subs = _ADDITION_SEP_RE.split(spec)
+ if not subs:
+ raise ValueError(f"No instructions could be built out of {spec}")
+ instruction = _str_to_read_instruction(subs[0])
+ return sum((_str_to_read_instruction(sub) for sub in subs[1:]), instruction)
+
+ def to_spec(self):
+ rel_instr_specs = []
+ for rel_instr in self._relative_instructions:
+ rel_instr_spec = rel_instr.splitname
+ if rel_instr.from_ is not None or rel_instr.to is not None:
+ from_ = rel_instr.from_
+ to = rel_instr.to
+ unit = rel_instr.unit
+ rounding = rel_instr.rounding
+ unit = unit if unit == "%" else ""
+ from_ = str(from_) + unit if from_ is not None else ""
+ to = str(to) + unit if to is not None else ""
+ slice_str = f"[{from_}:{to}]"
+ rounding_str = (
+ f"({rounding})" if unit == "%" and rounding is not None and rounding != "closest" else ""
+ )
+ rel_instr_spec += slice_str + rounding_str
+ rel_instr_specs.append(rel_instr_spec)
+ return "+".join(rel_instr_specs)
+
+ def __add__(self, other):
+ """Returns a new ReadInstruction obj, result of appending other to self."""
+ if not isinstance(other, ReadInstruction):
+ msg = "ReadInstruction can only be added to another ReadInstruction obj."
+ raise TypeError(msg)
+ self_ris = self._relative_instructions
+ other_ris = other._relative_instructions # pylint: disable=protected-access
+ if (
+ self_ris[0].unit != "abs"
+ and other_ris[0].unit != "abs"
+ and self._relative_instructions[0].rounding != other_ris[0].rounding
+ ):
+ raise ValueError("It is forbidden to sum ReadInstruction instances with different rounding values.")
+ return self._read_instruction_from_relative_instructions(self_ris + other_ris)
+
+ def __str__(self):
+ return self.to_spec()
+
+ def __repr__(self):
+ return f"ReadInstruction({self._relative_instructions})"
+
+ def to_absolute(self, name2len):
+ """Translate instruction into a list of absolute instructions.
+
+ Those absolute instructions are then to be added together.
+
+ Args:
+ name2len (`dict`):
+ Associating split names to number of examples.
+
+ Returns:
+ list of _AbsoluteInstruction instances (corresponds to the + in spec).
+ """
+ return [_rel_to_abs_instr(rel_instr, name2len) for rel_instr in self._relative_instructions]
diff --git a/venv/Lib/site-packages/datasets/arrow_writer.py b/venv/Lib/site-packages/datasets/arrow_writer.py
new file mode 100644
index 0000000000000000000000000000000000000000..2bd6f9fc6130e00256dbc384cae41020ba6f1314
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/arrow_writer.py
@@ -0,0 +1,828 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""To write records into Parquet files."""
+
+import io
+import json
+import sys
+from collections.abc import Iterable
+from typing import Any, Literal, Optional
+
+import fsspec
+import numpy as np
+import pyarrow as pa
+import pyarrow.json as paj
+import pyarrow.parquet as pq
+from fsspec.core import url_to_fs
+
+from . import config
+from .features import Audio, Features, Image, Pdf, Value, Video
+from .features.features import (
+ FeatureType,
+ List,
+ _ArrayXDExtensionType,
+ _visit,
+ cast_to_python_objects,
+ generate_from_arrow_type,
+ get_nested_type,
+ list_of_np_array_to_pyarrow_listarray,
+ numpy_to_pyarrow_listarray,
+ require_storage_embed,
+ to_pyarrow_listarray,
+)
+from .filesystems import is_remote_filesystem
+from .info import DatasetInfo
+from .table import array_cast, cast_array_to_feature, embed_table_storage, table_cast
+from .utils import logging
+from .utils.json import (
+ find_mixed_struct_types_field_paths,
+ get_json_field_path_from_pyarrow_json_error,
+ get_json_field_paths_from_feature,
+ insert_json_field_path,
+ json_encode_field,
+ json_encode_fields_in_json_lines,
+ set_json_types_in_feature,
+ ujson_dumps,
+)
+from .utils.py_utils import asdict, convert_file_size_to_int, first_non_null_non_empty_value
+
+
+logger = logging.get_logger(__name__)
+
+type_ = type # keep python's type function
+
+
+def get_arrow_writer_batch_size_from_features(features: Optional[Features]) -> Optional[int]:
+ """
+ Get the writer_batch_size that defines the maximum record batch size in the arrow files based on configuration values.
+ The default value is 100 for image/audio datasets and 10 for videos.
+ This allows to avoid overflows in arrow buffers.
+
+ Args:
+ features (`datasets.Features` or `None`):
+ Dataset Features from `datasets`.
+ Returns:
+ writer_batch_size (`Optional[int]`):
+ Writer batch size to pass to a dataset builder.
+ If `None`, then it will use the `datasets` default, i.e. `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+ """
+ if not features:
+ return None
+
+ batch_size = np.inf
+
+ def set_batch_size(feature: FeatureType) -> None:
+ nonlocal batch_size
+ if isinstance(feature, Image) and config.ARROW_RECORD_BATCH_SIZE_FOR_IMAGE_DATASETS is not None:
+ batch_size = min(batch_size, config.ARROW_RECORD_BATCH_SIZE_FOR_IMAGE_DATASETS)
+ elif isinstance(feature, Audio) and config.ARROW_RECORD_BATCH_SIZE_FOR_AUDIO_DATASETS is not None:
+ batch_size = min(batch_size, config.ARROW_RECORD_BATCH_SIZE_FOR_AUDIO_DATASETS)
+ elif isinstance(feature, Video) and config.ARROW_RECORD_BATCH_SIZE_FOR_VIDEO_DATASETS is not None:
+ batch_size = min(batch_size, config.ARROW_RECORD_BATCH_SIZE_FOR_VIDEO_DATASETS)
+ elif (
+ isinstance(feature, Value)
+ and feature.dtype == "binary"
+ and config.ARROW_RECORD_BATCH_SIZE_FOR_BINARY_DATASETS is not None
+ ):
+ batch_size = min(batch_size, config.ARROW_RECORD_BATCH_SIZE_FOR_BINARY_DATASETS)
+
+ _visit(features, set_batch_size)
+
+ return None if batch_size is np.inf else batch_size
+
+
+def get_writer_batch_size_from_features(features: Optional[Features]) -> Optional[int]:
+ """
+ Get the writer_batch_size that defines the maximum row group size in the parquet files based on configuration values.
+ By default these are not set, but it can be helpful to hard set those values in some cases.
+ This allows to optimize random access to parquet file, since accessing 1 row requires
+ to read its entire row group.
+
+ Args:
+ features (`datasets.Features` or `None`):
+ Dataset Features from `datasets`.
+ Returns:
+ writer_batch_size (`Optional[int]`):
+ Writer batch size to pass to a parquet writer.
+ If `None`, then it will use the `datasets` default, i.e. aiming for row groups of 100MB.
+ """
+ if not features:
+ return None
+
+ batch_size = np.inf
+
+ def set_batch_size(feature: FeatureType) -> None:
+ nonlocal batch_size
+ if isinstance(feature, Image) and config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS is not None:
+ batch_size = min(batch_size, config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS)
+ elif isinstance(feature, Audio) and config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS is not None:
+ batch_size = min(batch_size, config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS)
+ elif isinstance(feature, Video) and config.PARQUET_ROW_GROUP_SIZE_FOR_VIDEO_DATASETS is not None:
+ batch_size = min(batch_size, config.PARQUET_ROW_GROUP_SIZE_FOR_VIDEO_DATASETS)
+ elif (
+ isinstance(feature, Value)
+ and feature.dtype == "binary"
+ and config.PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS is not None
+ ):
+ batch_size = min(batch_size, config.PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS)
+
+ _visit(features, set_batch_size)
+
+ return None if batch_size is np.inf else batch_size
+
+
+def get_writer_batch_size_from_data_size(num_rows: int, num_bytes: int) -> int:
+ """
+ Get the writer_batch_size that defines the maximum row group size in the parquet files.
+ The default in `datasets` is aiming for row groups of maximum 100MB uncompressed.
+ This allows to optimize random access to parquet file, since accessing 1 row requires
+ to read its entire row group.
+
+ This can be improved to get optimized size for querying/iterating
+ but at least it matches the dataset viewer expectations on HF.
+
+ Args:
+ num_rows (`int`):
+ Number of rows in the dataset.
+ num_bytes (`int`):
+ Number of bytes in the dataset.
+ For dataset with external files to embed (image, audio, videos), this can also be an
+ estimate from `dataset._estimate_nbytes()`.
+ Returns:
+ writer_batch_size (`Optional[int]`):
+ Writer batch size to pass to a parquet writer.
+ """
+ return max(1, num_rows * convert_file_size_to_int(config.MAX_ROW_GROUP_SIZE) // num_bytes) if num_bytes > 0 else 1
+
+
+class SchemaInferenceError(ValueError):
+ pass
+
+
+class TypedSequence:
+ """
+ This data container generalizes the typing when instantiating pyarrow arrays, tables or batches.
+
+ More specifically it adds several features:
+ - Support extension types like ``datasets.features.Array2DExtensionType``:
+ By default pyarrow arrays don't return extension arrays. One has to call
+ ``pa.ExtensionArray.from_storage(type, pa.array(data, type.storage_type))``
+ in order to get an extension array.
+ - Support for ``try_type`` parameter that can be used instead of ``type``:
+ When an array is transformed, we like to keep the same type as before if possible.
+ For example when calling :func:`datasets.Dataset.map`, we don't want to change the type
+ of each column by default.
+ - Better error message when a pyarrow array overflows.
+
+ Example::
+
+ from datasets.features import Array2D, Array2DExtensionType, Value
+ from datasets.arrow_writer import TypedSequence
+ import pyarrow as pa
+
+ arr = pa.array(TypedSequence([1, 2, 3], type=Value("int32")))
+ assert arr.type == pa.int32()
+
+ arr = pa.array(TypedSequence([1, 2, 3], try_type=Value("int32")))
+ assert arr.type == pa.int32()
+
+ arr = pa.array(TypedSequence(["foo", "bar"], try_type=Value("int32")))
+ assert arr.type == pa.string()
+
+ arr = pa.array(TypedSequence([[[1, 2, 3]]], type=Array2D((1, 3), "int64")))
+ assert arr.type == Array2DExtensionType((1, 3), "int64")
+
+ table = pa.Table.from_pydict({
+ "image": TypedSequence([[[1, 2, 3]]], type=Array2D((1, 3), "int64"))
+ })
+ assert table["image"].type == Array2DExtensionType((1, 3), "int64")
+
+ """
+
+ def __init__(
+ self,
+ data: Iterable,
+ type: Optional[FeatureType] = None,
+ try_type: Optional[FeatureType] = None,
+ optimized_int_type: Optional[FeatureType] = None,
+ on_mixed_types: Optional[Literal["use_json"]] = None,
+ ):
+ # assert type is None or try_type is None,
+ if type is not None and try_type is not None:
+ raise ValueError("You cannot specify both type and try_type")
+ # set attributes
+ self.data = data
+ self.type = type
+ self.try_type = try_type # is ignored if it doesn't match the data
+ self.optimized_int_type = optimized_int_type
+ self.on_mixed_types = on_mixed_types
+ # when trying a type (is ignored if data is not compatible)
+ self.trying_type = self.try_type is not None
+ self.trying_int_optimization = optimized_int_type is not None and type is None and try_type is None
+ # used to get back the inferred type after __arrow_array__() is called once
+ self._inferred_type = None
+
+ def get_inferred_type(self) -> FeatureType:
+ """Return the inferred feature type.
+ This is done by converting the sequence to an Arrow array, and getting the corresponding
+ feature type.
+
+ Since building the Arrow array can be expensive, the value of the inferred type is cached
+ as soon as pa.array is called on the typed sequence.
+
+ Returns:
+ FeatureType: inferred feature type of the sequence.
+ """
+ if self._inferred_type is None:
+ pa.array(self)
+ return self._inferred_type
+
+ @staticmethod
+ def _infer_custom_type_and_encode(data: Iterable) -> tuple[Iterable, Optional[FeatureType]]:
+ """Implement type inference for custom objects like PIL.Image.Image -> Image type.
+
+ This function is only used for custom python objects that can't be directly passed to build
+ an Arrow array. In such cases is infers the feature type to use, and it encodes the data so
+ that they can be passed to an Arrow array.
+
+ Args:
+ data (Iterable): array of data to infer the type, e.g. a list of PIL images.
+
+ Returns:
+ Tuple[Iterable, Optional[FeatureType]]: a tuple with:
+ - the (possibly encoded) array, if the inferred feature type requires encoding
+ - the inferred feature type if the array is made of supported custom objects like
+ PIL images, else None.
+ """
+ if config.PIL_AVAILABLE and "PIL" in sys.modules:
+ import PIL.Image
+
+ non_null_idx, non_null_value = first_non_null_non_empty_value(data)
+ if isinstance(non_null_value, PIL.Image.Image):
+ return [Image().encode_example(value) if value is not None else None for value in data], Image()
+ if isinstance(non_null_value, list) and isinstance(non_null_value[0], PIL.Image.Image):
+ return [
+ [Image().encode_example(x) for x in value] if value is not None else None for value in data
+ ], List(Image())
+ if config.PDFPLUMBER_AVAILABLE and "pdfplumber" in sys.modules:
+ import pdfplumber
+
+ non_null_idx, non_null_value = first_non_null_non_empty_value(data)
+ if isinstance(non_null_value, pdfplumber.pdf.PDF):
+ return [Pdf().encode_example(value) if value is not None else None for value in data], Pdf()
+ if isinstance(non_null_value, list) and isinstance(non_null_value[0], pdfplumber.pdf.PDF):
+ return [
+ [Pdf().encode_example(x) for x in value] if value is not None else None for value in data
+ ], List(Pdf())
+ return data, None
+
+ def __arrow_array__(self, type: Optional[pa.DataType] = None):
+ out = self._arrow_array(type=type)
+ if self._inferred_type is None:
+ self._inferred_type = generate_from_arrow_type(out.type)
+ return out
+
+ def _arrow_array(self, type: Optional[pa.DataType] = None):
+ """This function is called when calling pa.array(typed_sequence)"""
+
+ if type is not None:
+ raise ValueError("TypedSequence is supposed to be used with pa.array(typed_sequence, type=None)")
+ del type # make sure we don't use it
+ data = self.data
+ # automatic type inference for custom objects
+ if self.type is None and self.try_type is None:
+ data, self._inferred_type = self._infer_custom_type_and_encode(data)
+ if self._inferred_type is None:
+ type = self.try_type if self.trying_type else self.type
+ else:
+ type = self._inferred_type
+ pa_type = get_nested_type(type) if type is not None else None
+ optimized_int_pa_type = (
+ get_nested_type(self.optimized_int_type) if self.optimized_int_type is not None else None
+ )
+ trying_cast_to_python_objects = False
+ json_field_paths = []
+ try:
+ # custom pyarrow types
+ if isinstance(pa_type, _ArrayXDExtensionType):
+ storage = to_pyarrow_listarray(data, pa_type)
+ return pa.ExtensionArray.from_storage(pa_type, storage)
+
+ # efficient np array to pyarrow array
+ if isinstance(data, np.ndarray):
+ out = numpy_to_pyarrow_listarray(data)
+ elif isinstance(data, list) and data and isinstance(first_non_null_non_empty_value(data)[1], np.ndarray):
+ out = list_of_np_array_to_pyarrow_listarray(data)
+ else:
+ trying_cast_to_python_objects = True
+ examples = data
+ # find fields to json-encode
+ if self.on_mixed_types == "use_json" and type is None:
+ json_field_paths = find_mixed_struct_types_field_paths(examples, allow_root=True)
+ elif type is not None:
+ json_field_paths = get_json_field_paths_from_feature(type)
+ # json encode if needed
+ if json_field_paths:
+ for json_field_path in json_field_paths:
+ examples = [json_encode_field(examples, json_field_path) for examples in examples]
+ # to arrow array
+ out = pa.array(cast_to_python_objects(examples, only_1d_for_numpy=True))
+ # cast to json type if needed
+ if json_field_paths:
+ pa_table = pa.Table.from_arrays([out], names=["obj"])
+ features = Features.from_arrow_schema(pa_table.schema)
+ feature = set_json_types_in_feature(features["obj"], json_field_paths)
+ pa_table = table_cast(pa_table, Features({"obj": feature}).arrow_schema)
+ out = pa_table[0] # get the "obj" column
+ # use smaller integer precisions if possible
+ if self.trying_int_optimization:
+ if pa.types.is_int64(out.type):
+ out = out.cast(optimized_int_pa_type)
+ elif pa.types.is_list(out.type):
+ if pa.types.is_int64(out.type.value_type):
+ out = array_cast(out, pa.list_(optimized_int_pa_type))
+ elif pa.types.is_list(out.type.value_type) and pa.types.is_int64(out.type.value_type.value_type):
+ out = array_cast(out, pa.list_(pa.list_(optimized_int_pa_type)))
+ # otherwise we can finally use the user's type
+ elif type is not None:
+ # We use cast_array_to_feature to support casting to custom types like Audio and Image
+ # Also, when trying type "string", we don't want to convert integers or floats to "string".
+ # We only do it if trying_type is False - since this is what the user asks for.
+ out = cast_array_to_feature(
+ out, type, allow_primitive_to_str=not self.trying_type, allow_decimal_to_str=not self.trying_type
+ )
+ return out
+ except (
+ TypeError,
+ pa.lib.ArrowTypeError,
+ pa.lib.ArrowInvalid,
+ pa.lib.ArrowNotImplementedError,
+ ) as e: # handle type errors and overflows
+ # Ignore ArrowNotImplementedError caused by trying type, otherwise re-raise
+ if not self.trying_type and isinstance(e, pa.lib.ArrowNotImplementedError):
+ raise
+
+ if self.trying_type:
+ try: # second chance
+ if isinstance(data, np.ndarray):
+ return numpy_to_pyarrow_listarray(data)
+ elif isinstance(data, list) and data and any(isinstance(value, np.ndarray) for value in data):
+ return list_of_np_array_to_pyarrow_listarray(data)
+ else:
+ trying_cast_to_python_objects = True
+ return pa.array(cast_to_python_objects(data, only_1d_for_numpy=True))
+ except pa.lib.ArrowInvalid as e:
+ if "overflow" in str(e):
+ raise OverflowError(
+ f"There was an overflow with type {type_(data)}. Try to reduce writer_batch_size to have batches smaller than 2GB.\n({e})"
+ ) from None
+ elif self.trying_int_optimization and "not in range" in str(e):
+ optimized_int_pa_type_str = np.dtype(optimized_int_pa_type.to_pandas_dtype()).name
+ logger.info(
+ f"Failed to cast a sequence to {optimized_int_pa_type_str}. Falling back to int64."
+ )
+ return out
+ elif trying_cast_to_python_objects and "Could not convert" in str(e):
+ out = pa.array(
+ cast_to_python_objects(data, only_1d_for_numpy=True, optimize_list_casting=False)
+ )
+ if type is not None:
+ out = cast_array_to_feature(
+ out, type, allow_primitive_to_str=True, allow_decimal_to_str=True
+ )
+ return out
+ else:
+ raise
+ elif "overflow" in str(e):
+ raise OverflowError(
+ f"There was an overflow with type {type_(data)}. Try to reduce writer_batch_size to have batches smaller than 2GB.\n({e})"
+ ) from None
+ elif self.trying_int_optimization and "not in range" in str(e):
+ optimized_int_pa_type_str = np.dtype(optimized_int_pa_type.to_pandas_dtype()).name
+ logger.info(f"Failed to cast a sequence to {optimized_int_pa_type_str}. Falling back to int64.")
+ return out
+ elif trying_cast_to_python_objects and (
+ "Could not convert" in str(e) or "cannot mix struct and non-struct" in str(e) or "Expected " in str(e)
+ ):
+ try: # third chance
+ out = pa.array(cast_to_python_objects(data, only_1d_for_numpy=True, optimize_list_casting=False))
+ except (pa.ArrowInvalid, pa.ArrowTypeError) as ee:
+ # in case of mixed types, we use the JSON Lines reader in pyarrow to locate them and set them to json fields
+ if self.on_mixed_types == "use_json" and (
+ "Could not convert " in str(ee)
+ or "cannot mix struct and non-struct" in str(ee)
+ or "Expected " in str(ee)
+ ):
+ # we use "obj" to have valid JSON Lines since data may contain lists
+ original_batch = "\n".join([ujson_dumps({"obj": example}) for example in data]).encode()
+ json_field_paths = [["obj"] + json_field_path for json_field_path in json_field_paths]
+ batch = json_encode_fields_in_json_lines(original_batch, json_field_paths)
+ pa_table = None
+ while True:
+ try: # fourth chance
+ pa_table = paj.read_json(
+ io.BytesIO(batch), read_options=paj.ReadOptions(use_threads=False)
+ )
+ break
+ except pa.ArrowInvalid as eee:
+ if "JSON parse error: Column(" in str(eee) and ") changed from" in str(eee):
+ json_field_path = get_json_field_path_from_pyarrow_json_error(str(eee))
+ insert_json_field_path(json_field_paths, json_field_path)
+ batch = json_encode_fields_in_json_lines(original_batch, json_field_paths)
+ else:
+ break
+ if pa_table is not None:
+ features = Features.from_arrow_schema(pa_table.schema)
+ features = set_json_types_in_feature(features, json_field_paths)
+ pa_table = table_cast(pa_table, features.arrow_schema)
+ out = pa_table[0] # get the "obj" column
+ return out
+ else:
+ raise
+ else:
+ raise
+ if type is not None:
+ out = cast_array_to_feature(out, type, allow_primitive_to_str=True, allow_decimal_to_str=True)
+ return out
+ else:
+ raise
+
+
+class OptimizedTypedSequence(TypedSequence):
+ def __init__(
+ self,
+ data,
+ type: Optional[FeatureType] = None,
+ try_type: Optional[FeatureType] = None,
+ col: Optional[str] = None,
+ optimized_int_type: Optional[FeatureType] = None,
+ on_mixed_types: Optional[Literal["use_json"]] = None,
+ ):
+ optimized_int_type_by_col = {
+ "attention_mask": Value("int8"), # binary tensor
+ "special_tokens_mask": Value("int8"),
+ "input_ids": Value("int32"), # typical vocab size: 0-50k (max ~500k, never > 1M)
+ "token_type_ids": Value(
+ "int8"
+ ), # binary mask; some (XLNetModel) use an additional token represented by a 2
+ }
+ if type is None and try_type is None:
+ optimized_int_type = optimized_int_type_by_col.get(col, None)
+ super().__init__(
+ data, type=type, try_type=try_type, optimized_int_type=optimized_int_type, on_mixed_types=on_mixed_types
+ )
+
+
+class ArrowWriter:
+ """Shuffles and writes Examples to Arrow files."""
+
+ def __init__(
+ self,
+ schema: Optional[pa.Schema] = None,
+ features: Optional[Features] = None,
+ path: Optional[str] = None,
+ stream: Optional[pa.NativeFile] = None,
+ fingerprint: Optional[str] = None,
+ writer_batch_size: Optional[int] = None,
+ disable_nullable: bool = False,
+ update_features: bool = False,
+ on_mixed_types: Optional[Literal["use_json"]] = "use_json",
+ with_metadata: bool = True,
+ unit: str = "examples",
+ embed_local_files: bool = False,
+ storage_options: Optional[dict] = None,
+ ):
+ if path is None and stream is None:
+ raise ValueError("At least one of path and stream must be provided.")
+ if features is not None:
+ self._features = features
+ self._schema = None
+ elif schema is not None:
+ self._schema: pa.Schema = schema
+ self._features = Features.from_arrow_schema(self._schema)
+ else:
+ self._features = None
+ self._schema = None
+
+ self._disable_nullable = disable_nullable
+
+ if stream is None:
+ fs, path = url_to_fs(path, **(storage_options or {}))
+ self._fs: fsspec.AbstractFileSystem = fs
+ self._path = path if not is_remote_filesystem(self._fs) else self._fs.unstrip_protocol(path)
+ self.stream = self._fs.open(path, "wb")
+ self._closable_stream = True
+ else:
+ self._fs = None
+ self._path = None
+ self.stream = stream
+ self._closable_stream = False
+
+ self.fingerprint = fingerprint
+ self.disable_nullable = disable_nullable
+ self.writer_batch_size = (
+ writer_batch_size
+ or get_arrow_writer_batch_size_from_features(self._features)
+ or config.DEFAULT_MAX_BATCH_SIZE
+ )
+ self.update_features = update_features
+ self.on_mixed_types = on_mixed_types
+ self.with_metadata = with_metadata
+ self.unit = unit
+ self.embed_local_files = embed_local_files
+
+ self._num_examples = 0
+ self._num_bytes = 0
+ self.current_examples: list[tuple[dict[str, Any], str]] = []
+ self.current_rows: list[pa.Table] = []
+ self.pa_writer: Optional[pa.RecordBatchStreamWriter] = None
+ self.hkey_record = []
+
+ def __len__(self):
+ """Return the number of writed and staged examples"""
+ return self._num_examples + len(self.current_examples) + len(self.current_rows)
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.close()
+
+ def close(self):
+ # Try closing if opened; if closed: pyarrow.lib.ArrowInvalid: Invalid operation on closed file
+ if self.pa_writer: # it might be None
+ try:
+ self.pa_writer.close()
+ except Exception: # pyarrow.lib.ArrowInvalid, OSError
+ pass
+ if self._closable_stream and not self.stream.closed:
+ self.stream.close() # This also closes self.pa_writer if it is opened
+
+ def _build_schema(self, inferred_schema: pa.Schema):
+ schema = self.schema
+ features = self._features
+ inferred_features = Features.from_arrow_schema(inferred_schema)
+ if self._features is not None:
+ if self.update_features: # keep original features it they match, or update them
+ fields = {field.name: field for field in self._features.type}
+ for inferred_field in inferred_features.type:
+ name = inferred_field.name
+ if name in fields:
+ if inferred_field == fields[name]:
+ inferred_features[name] = self._features[name]
+ features = inferred_features
+ schema: pa.Schema = inferred_schema
+ else:
+ features = inferred_features
+ schema: pa.Schema = inferred_features.arrow_schema
+
+ if self.disable_nullable:
+ schema = pa.schema(pa.field(field.name, field.type, nullable=False) for field in schema)
+ if self.with_metadata:
+ schema = schema.with_metadata(self._build_metadata(DatasetInfo(features=features), self.fingerprint))
+ else:
+ schema = schema.with_metadata({})
+
+ return schema, features
+
+ def _build_writer(self, inferred_schema: pa.Schema):
+ self._schema, self._features = self._build_schema(inferred_schema)
+ self.pa_writer = pa.RecordBatchStreamWriter(self.stream, self._schema)
+
+ @property
+ def schema(self):
+ _schema = (
+ self._schema
+ if self._schema is not None
+ else (pa.schema(self._features.type) if self._features is not None else None)
+ )
+ if self._disable_nullable and _schema is not None:
+ _schema = pa.schema(pa.field(field.name, field.type, nullable=False) for field in _schema)
+ return _schema if _schema is not None else []
+
+ @staticmethod
+ def _build_metadata(info: DatasetInfo, fingerprint: Optional[str] = None) -> dict[str, str]:
+ info_keys = ["features"] # we can add support for more DatasetInfo keys in the future
+ info_as_dict = asdict(info)
+ metadata = {}
+ metadata["info"] = {key: info_as_dict[key] for key in info_keys}
+ if fingerprint is not None:
+ metadata["fingerprint"] = fingerprint
+ return {"huggingface": json.dumps(metadata)}
+
+ def write_examples_on_file(self):
+ """Write stored examples from the write-pool of examples. It makes a table out of the examples and write it."""
+ if not self.current_examples:
+ return
+ # preserve the order the columns
+ if self.schema:
+ schema_cols = set(self.schema.names)
+ examples_cols = self.current_examples[0][0].keys() # .keys() preserves the order (unlike set)
+ common_cols = [col for col in self.schema.names if col in examples_cols]
+ extra_cols = [col for col in examples_cols if col not in schema_cols]
+ cols = common_cols + extra_cols
+ else:
+ cols = list(self.current_examples[0][0])
+ batch_examples = {}
+ for col in cols:
+ # We use row[0][col] since current_examples contains (example, key) tuples.
+ # Moreover, examples could be Arrow arrays of 1 element.
+ # This can happen in `.map()` when we want to re-write the same Arrow data
+ if all(isinstance(row[0][col], (pa.Array, pa.ChunkedArray)) for row in self.current_examples):
+ arrays = [row[0][col] for row in self.current_examples]
+ arrays = [
+ chunk
+ for array in arrays
+ for chunk in (array.chunks if isinstance(array, pa.ChunkedArray) else [array])
+ ]
+ batch_examples[col] = pa.concat_arrays(arrays)
+ else:
+ batch_examples[col] = [
+ row[0][col].to_pylist()[0] if isinstance(row[0][col], (pa.Array, pa.ChunkedArray)) else row[0][col]
+ for row in self.current_examples
+ ]
+ self._write_batch(batch_examples=batch_examples)
+ self.current_examples = []
+
+ def write_rows_on_file(self):
+ """Write stored rows from the write-pool of rows. It concatenates the single-row tables and it writes the resulting table."""
+ if not self.current_rows:
+ return
+ table = pa.concat_tables(self.current_rows)
+ self._write_table(table)
+ self.current_rows = []
+
+ def write(
+ self,
+ example: dict[str, Any],
+ writer_batch_size: Optional[int] = None,
+ ):
+ """Add a given (Example,Key) pair to the write-pool of examples which is written to file.
+
+ Args:
+ example: the Example to add.
+ """
+ # Store example as a tuple so as to keep the structure of `self.current_examples` uniform
+ self.current_examples.append((example, ""))
+
+ if writer_batch_size is None:
+ writer_batch_size = self.writer_batch_size
+ if writer_batch_size is not None and len(self.current_examples) >= writer_batch_size:
+ self.write_examples_on_file()
+
+ def write_row(self, row: pa.Table, writer_batch_size: Optional[int] = None):
+ """Add a given single-row Table to the write-pool of rows which is written to file.
+
+ Args:
+ row: the row to add.
+ """
+ if len(row) != 1:
+ raise ValueError(f"Only single-row pyarrow tables are allowed but got table with {len(row)} rows.")
+ self.current_rows.append(row)
+ if writer_batch_size is None:
+ writer_batch_size = self.writer_batch_size
+ if writer_batch_size is not None and len(self.current_rows) >= writer_batch_size:
+ self.write_rows_on_file()
+
+ def write_batch(
+ self,
+ batch_examples: dict[str, list],
+ writer_batch_size: Optional[int] = None,
+ try_original_type: Optional[bool] = True,
+ ):
+ """Write a batch of Example to file.
+ Ignores the batch if it appears to be empty,
+ preventing a potential schema update of unknown types.
+
+ Args:
+ batch_examples: the batch of examples to add.
+ try_original_type: use `try_type` when instantiating OptimizedTypedSequence if `True`, otherwise `try_type = None`.
+ """
+ self.write_examples_on_file() # in case there are buffered examples to write first
+ self._write_batch(batch_examples, writer_batch_size=writer_batch_size, try_original_type=try_original_type)
+
+ def _write_batch(
+ self,
+ batch_examples: dict[str, list],
+ writer_batch_size: Optional[int] = None,
+ try_original_type: Optional[bool] = True,
+ ):
+ if batch_examples and len(next(iter(batch_examples.values()))) == 0:
+ return
+ features = None if self.pa_writer is None and self.update_features else self._features
+ try_features = self._features if self.pa_writer is None and self.update_features else None
+ arrays = []
+ inferred_features = Features()
+ # preserve the order the columns
+ if self.schema:
+ schema_cols = set(self.schema.names)
+ batch_cols = batch_examples.keys() # .keys() preserves the order (unlike set)
+ common_cols = [col for col in self.schema.names if col in batch_cols]
+ extra_cols = [col for col in batch_cols if col not in schema_cols]
+ cols = common_cols + extra_cols
+ else:
+ cols = list(batch_examples)
+ for col in cols:
+ col_values = batch_examples[col]
+ col_type = features[col] if features else None
+ if isinstance(col_values, (pa.Array, pa.ChunkedArray)):
+ array = cast_array_to_feature(col_values, col_type) if col_type is not None else col_values
+ arrays.append(array)
+ inferred_features[col] = generate_from_arrow_type(col_values.type)
+ else:
+ col_try_type = (
+ try_features[col]
+ if try_features is not None and col in try_features and try_original_type
+ else None
+ )
+ typed_sequence = OptimizedTypedSequence(
+ col_values, type=col_type, try_type=col_try_type, col=col, on_mixed_types=self.on_mixed_types
+ )
+ arrays.append(pa.array(typed_sequence))
+ inferred_features[col] = typed_sequence.get_inferred_type()
+ schema = inferred_features.arrow_schema if self.pa_writer is None else self.schema
+ pa_table = pa.Table.from_arrays(arrays, schema=schema)
+ self.write_table(pa_table, writer_batch_size)
+
+ def write_table(self, pa_table: pa.Table, writer_batch_size: Optional[int] = None):
+ """Write a Table to file.
+
+ Args:
+ example: the Table to add.
+ """
+ self.write_rows_on_file() # in case there are buffered rows to write first
+ self._write_table(pa_table, writer_batch_size=writer_batch_size)
+
+ def _write_table(self, pa_table: pa.Table, writer_batch_size: Optional[int] = None):
+ if writer_batch_size is None:
+ writer_batch_size = self.writer_batch_size
+ if self.pa_writer is None:
+ self._build_writer(inferred_schema=pa_table.schema)
+ pa_table = pa_table.combine_chunks()
+ pa_table = table_cast(pa_table, self._schema)
+ if self.embed_local_files:
+ pa_table = embed_table_storage(pa_table, local_files=True, remote_files=False)
+ self._num_bytes += pa_table.nbytes
+ self._num_examples += pa_table.num_rows
+ self.pa_writer.write_table(pa_table, writer_batch_size)
+
+ def finalize(self, close_stream=True):
+ self.write_rows_on_file()
+ # In case current_examples < writer_batch_size, but user uses finalize()
+ self.write_examples_on_file()
+ # If schema is known, infer features even if no examples were written
+ if self.pa_writer is None and self.schema:
+ self._build_writer(self.schema)
+ if self.pa_writer is not None:
+ self.pa_writer.close()
+ self.pa_writer = None
+ if close_stream:
+ self.stream.close()
+ else:
+ if close_stream:
+ self.stream.close()
+ raise SchemaInferenceError("Please pass `features` or at least one example when writing data")
+ logger.debug(
+ f"Done writing {self._num_examples} {self.unit} in {self._num_bytes} bytes {self._path if self._path else ''}."
+ )
+ return self._num_examples, self._num_bytes
+
+
+class ParquetWriter(ArrowWriter):
+ def __init__(self, *args, use_content_defined_chunking=True, write_page_index=True, **kwargs):
+ super().__init__(*args, **kwargs)
+ if use_content_defined_chunking is True:
+ use_content_defined_chunking = config.DEFAULT_CDC_OPTIONS
+ self.use_content_defined_chunking = use_content_defined_chunking
+ self.write_page_index = write_page_index
+
+ def _build_writer(self, inferred_schema: pa.Schema):
+ self._schema, self._features = self._build_schema(inferred_schema)
+ self.pa_writer = pq.ParquetWriter(
+ self.stream,
+ self._schema,
+ use_content_defined_chunking=self.use_content_defined_chunking,
+ write_page_index=self.write_page_index,
+ compression={
+ col: "none" if require_storage_embed(feature) else "snappy" for col, feature in self._features.items()
+ },
+ use_dictionary=[col for col, feature in self._features.items() if not require_storage_embed(feature)],
+ column_encoding={
+ col: "PLAIN" for col, feature in self._features.items() if require_storage_embed(feature)
+ },
+ )
+ if self.use_content_defined_chunking is not False:
+ self.pa_writer.add_key_value_metadata(
+ {"content_defined_chunking": json.dumps(self.use_content_defined_chunking)}
+ )
diff --git a/venv/Lib/site-packages/datasets/builder.py b/venv/Lib/site-packages/datasets/builder.py
new file mode 100644
index 0000000000000000000000000000000000000000..ab677af89f02ae60fa1ea23d483dcae5aada3b2f
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/builder.py
@@ -0,0 +1,1916 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""DatasetBuilder base class."""
+
+import abc
+import contextlib
+import copy
+import inspect
+import os
+import posixpath
+import shutil
+import time
+import urllib
+from collections.abc import Iterator
+from dataclasses import dataclass
+from functools import partial
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Optional, Union
+from unittest.mock import patch
+
+import fsspec
+import pyarrow as pa
+from fsspec.core import url_to_fs
+from multiprocess import Pool
+from tqdm.contrib.concurrent import thread_map
+
+from . import config, utils
+from .arrow_dataset import Dataset
+from .arrow_reader import (
+ ArrowReader,
+ ReadInstruction,
+)
+from .arrow_writer import ArrowWriter, ParquetWriter, SchemaInferenceError
+from .data_files import DataFilesDict, DataFilesPatternsDict, sanitize_patterns
+from .dataset_dict import DatasetDict, IterableDatasetDict
+from .download.download_config import DownloadConfig
+from .download.download_manager import DownloadManager, DownloadMode
+from .download.streaming_download_manager import StreamingDownloadManager, xjoin
+from .exceptions import DatasetGenerationCastError, DatasetGenerationError, FileFormatError
+from .features import Features
+from .filesystems import (
+ is_remote_filesystem,
+ rename,
+)
+from .fingerprint import Hasher
+from .info import DatasetInfo
+from .iterable_dataset import (
+ ArrowExamplesIterable,
+ ExamplesIterable,
+ IterableDataset,
+ _concatenate_iterable_datasets,
+)
+from .naming import INVALID_WINDOWS_CHARACTERS_IN_PATH, camelcase_to_snakecase
+from .splits import Split, SplitDict, SplitGenerator, SplitInfo
+from .streaming import extend_dataset_builder_for_streaming
+from .table import CastError
+from .utils import logging
+from .utils import tqdm as hf_tqdm
+from .utils._filelock import FileLock
+from .utils.file_utils import is_remote_url
+from .utils.info_utils import VerificationMode, verify_checksums, verify_splits
+from .utils.py_utils import (
+ classproperty,
+ convert_file_size_to_int,
+ has_sufficient_disk_space,
+ iflatmap_unordered,
+ map_nested,
+ memoize,
+ size_str,
+ temporary_assignment,
+)
+from .utils.sharding import _number_of_shards_in_gen_kwargs, _split_gen_kwargs
+from .utils.track import tracked_list
+
+
+if TYPE_CHECKING:
+ from .load import DatasetModule
+
+
+logger = logging.get_logger(__name__)
+
+
+class InvalidConfigName(ValueError):
+ pass
+
+
+@dataclass
+class BuilderConfig:
+ """Base class for `DatasetBuilder` data configuration.
+
+ `DatasetBuilder` subclasses with data configuration options should subclass
+ `BuilderConfig` and add their own properties.
+
+ Attributes:
+ name (`str`, defaults to `default`):
+ The name of the configuration.
+ version (`Version` or `str`, defaults to `0.0.0`):
+ The version of the configuration.
+ data_dir (`str`, *optional*):
+ Path to the directory containing the source data.
+ data_files (`str` or `Sequence` or `Mapping`, *optional*):
+ Path(s) to source data file(s).
+ description (`str`, *optional*):
+ A human description of the configuration.
+ """
+
+ name: str = "default"
+ version: Optional[Union[utils.Version, str]] = utils.Version("0.0.0")
+ data_dir: Optional[str] = None
+ data_files: Optional[Union[DataFilesDict, DataFilesPatternsDict]] = None
+ description: Optional[str] = None
+
+ def __post_init__(self):
+ # The config name is used to name the cache directory.
+ for invalid_char in INVALID_WINDOWS_CHARACTERS_IN_PATH:
+ if invalid_char in self.name:
+ raise InvalidConfigName(
+ f"Bad characters from black list '{INVALID_WINDOWS_CHARACTERS_IN_PATH}' found in '{self.name}'. "
+ f"They could create issues when creating a directory for this config on Windows filesystem."
+ )
+ if self.data_files is not None and not isinstance(self.data_files, (DataFilesDict, DataFilesPatternsDict)):
+ raise ValueError(f"Expected a DataFilesDict in data_files but got {self.data_files}")
+
+ def __eq__(self, o):
+ # we need to override the default dataclass __eq__ since it doesn't check for
+ # other attributes that the ones of the signature.
+ if set(self.__dict__.keys()) != set(o.__dict__.keys()):
+ return False
+ return all((k, getattr(self, k)) == (k, getattr(o, k)) for k in self.__dict__.keys())
+
+ def create_config_id(
+ self,
+ config_kwargs: dict,
+ custom_features: Optional[Features] = None,
+ ) -> str:
+ """
+ The config id is used to build the cache directory.
+ By default it is equal to the config name.
+ However the name of a config is not sufficient to have a unique identifier for the dataset being generated
+ since it doesn't take into account:
+ - the config kwargs that can be used to overwrite attributes
+ - the custom features used to write the dataset
+ - the data_files for json/text/csv/pandas datasets
+
+ Therefore the config id is just the config name with an optional suffix based on these.
+ """
+ # Possibly add a suffix to the name to handle custom features/data_files/config_kwargs
+ suffix: Optional[str] = None
+ config_kwargs_to_add_to_suffix = config_kwargs.copy()
+ # name and version are already used to build the cache directory
+ config_kwargs_to_add_to_suffix.pop("name", None)
+ config_kwargs_to_add_to_suffix.pop("version", None)
+ # data dir handling (when specified it points to the manually downloaded data):
+ # it was previously ignored before the introduction of config id because we didn't want
+ # to change the config name. Now it's fine to take it into account for the config id.
+ # config_kwargs_to_add_to_suffix.pop("data_dir", None)
+ if "data_dir" in config_kwargs_to_add_to_suffix:
+ if config_kwargs_to_add_to_suffix["data_dir"] is None:
+ config_kwargs_to_add_to_suffix.pop("data_dir", None)
+ else:
+ # canonicalize the data dir to avoid two paths to the same location having different
+ # hashes
+ data_dir = config_kwargs_to_add_to_suffix["data_dir"]
+ data_dir = os.path.normpath(data_dir)
+ config_kwargs_to_add_to_suffix["data_dir"] = data_dir
+ if config_kwargs_to_add_to_suffix:
+ # we don't care about the order of the kwargs
+ config_kwargs_to_add_to_suffix = {
+ k: config_kwargs_to_add_to_suffix[k] for k in sorted(config_kwargs_to_add_to_suffix)
+ }
+ if all(isinstance(v, (str, bool, int, float)) for v in config_kwargs_to_add_to_suffix.values()):
+ suffix = ",".join(
+ str(k) + "=" + urllib.parse.quote_plus(str(v)) for k, v in config_kwargs_to_add_to_suffix.items()
+ )
+ if len(suffix) > 32: # hash if too long
+ suffix = Hasher.hash(config_kwargs_to_add_to_suffix)
+ else:
+ suffix = Hasher.hash(config_kwargs_to_add_to_suffix)
+
+ if custom_features is not None:
+ m = Hasher()
+ if suffix:
+ m.update(suffix)
+ m.update(custom_features)
+ suffix = m.hexdigest()
+
+ if suffix:
+ config_id = self.name + "-" + suffix
+ if len(config_id) > config.MAX_DATASET_CONFIG_ID_READABLE_LENGTH:
+ config_id = self.name + "-" + Hasher.hash(suffix)
+ return config_id
+ else:
+ return self.name
+
+ def _resolve_data_files(self, base_path: str, download_config: DownloadConfig) -> None:
+ if isinstance(self.data_files, DataFilesPatternsDict):
+ base_path = xjoin(base_path, self.data_dir) if self.data_dir else base_path
+ self.data_files = self.data_files.resolve(base_path, download_config)
+
+
+class DatasetBuilder:
+ """Abstract base class for all datasets.
+
+ `DatasetBuilder` has 3 key methods:
+
+ - [`DatasetBuilder.info`]: Documents the dataset, including feature
+ names, types, shapes, version, splits, citation, etc.
+ - [`DatasetBuilder.download_and_prepare`]: Downloads the source data
+ and writes it to disk.
+ - [`DatasetBuilder.as_dataset`]: Generates a [`Dataset`].
+
+ Some `DatasetBuilder`s expose multiple variants of the
+ dataset by defining a [`BuilderConfig`] subclass and accepting a
+ config object (or name) on construction. Configurable datasets expose a
+ pre-defined set of configurations in [`DatasetBuilder.builder_configs`].
+
+ Args:
+ cache_dir (`str`, *optional*):
+ Directory to cache data. Defaults to `"~/.cache/huggingface/datasets"`.
+ dataset_name (`str`, *optional*):
+ Name of the dataset, if different from the builder name. Useful for packaged builders
+ like csv, imagefolder, audiofolder, etc. to reflect the difference between datasets
+ that use the same packaged builder.
+ config_name (`str`, *optional*):
+ Name of the dataset configuration.
+ It affects the data generated on disk. Different configurations will have their own subdirectories and
+ versions.
+ If not provided, the default configuration is used (if it exists).
+
+
+
+ Parameter `name` was renamed to `config_name`.
+
+
+ hash (`str`, *optional*):
+ Hash specific to the dataset builder code. Used to update the caching directory when the
+ dataset builder code is updated (to avoid reusing old data).
+ The typical caching directory (defined in `self._relative_data_dir`) is `name/version/hash/`.
+ base_path (`str`, *optional*):
+ Base path for relative paths that are used to download files.
+ This can be a remote URL.
+ features ([`Features`], *optional*):
+ Features types to use with this dataset.
+ It can be used to change the [`Features`] types of a dataset, for example.
+ token (`str` or `bool`, *optional*):
+ String or boolean to use as Bearer token for remote files on the
+ Datasets Hub. If `True`, will get token from `"~/.huggingface"`.
+ repo_id (`str`, *optional*):
+ ID of the dataset repository.
+ Used to distinguish builders with the same name but not coming from the same namespace, for example "rajpurkar/squad"
+ and "lhoestq/squad" repo IDs. In the latter, the builder name would be "lhoestq___squad".
+ data_files (`str` or `Sequence` or `Mapping`, *optional*):
+ Path(s) to source data file(s).
+ For builders like "csv" or "json" that need the user to specify data files. They can be either
+ local or remote files. For convenience, you can use a `DataFilesDict`.
+ data_dir (`str`, *optional*):
+ Path to directory containing source data file(s).
+ Use only if `data_files` is not passed, in which case it is equivalent to passing
+ `os.path.join(data_dir, "**")` as `data_files`.
+ For builders that require manual download, it must be the path to the local directory containing the
+ manually downloaded data.
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the dataset file-system backend, if any.
+ writer_batch_size (`int`, *optional*):
+ Batch size used by the ArrowWriter.
+ It defines the number of samples that are kept in memory before writing them
+ and also the length of the arrow chunks.
+ None means that the ArrowWriter will use its default value.
+ **config_kwargs (additional keyword arguments): Keyword arguments to be passed to the corresponding builder
+ configuration class, set on the class attribute [`DatasetBuilder.BUILDER_CONFIG_CLASS`]. The builder
+ configuration class is [`BuilderConfig`] or a subclass of it.
+ """
+
+ # Default version
+ VERSION = None # Default version set in BuilderConfig
+
+ # Class for the builder config.
+ BUILDER_CONFIG_CLASS = BuilderConfig
+
+ # Named configurations that modify the data generated by download_and_prepare.
+ BUILDER_CONFIGS = []
+
+ # Optional default config name to be used when name is None
+ DEFAULT_CONFIG_NAME = None
+
+ # Default batch size used by the ArrowWriter
+ # It defines the number of samples that are kept in memory before writing them
+ # and also the length of the arrow chunks
+ # None means that the ArrowWriter will use its default value
+ DEFAULT_WRITER_BATCH_SIZE = None
+
+ # Useful to make sure PyArrow threads in c++ have time to shut down before gargabe collection
+ SLEEP_ON_THREADS_SHUTDOWNS = False
+
+ def __init__(
+ self,
+ cache_dir: Optional[str] = None,
+ dataset_name: Optional[str] = None,
+ config_name: Optional[str] = None,
+ hash: Optional[str] = None,
+ base_path: Optional[str] = None,
+ info: Optional[DatasetInfo] = None,
+ features: Optional[Features] = None,
+ token: Optional[Union[bool, str]] = None,
+ repo_id: Optional[str] = None,
+ data_files: Optional[Union[str, list, dict, DataFilesDict]] = None,
+ data_dir: Optional[str] = None,
+ storage_options: Optional[dict] = None,
+ writer_batch_size: Optional[int] = None,
+ config_id: Optional[str] = None,
+ **config_kwargs,
+ ):
+ # DatasetBuilder name
+ self.name: str = camelcase_to_snakecase(self.__module__.split(".")[-1])
+ self.hash: Optional[str] = hash
+ self.base_path = base_path
+ self.token = token
+ self.repo_id = repo_id
+ self.storage_options = storage_options or {}
+ self.dataset_name = camelcase_to_snakecase(dataset_name) if dataset_name else self.name
+ self._writer_batch_size = writer_batch_size or self.DEFAULT_WRITER_BATCH_SIZE
+
+ if data_files is not None and not isinstance(data_files, DataFilesDict):
+ data_files = DataFilesDict.from_patterns(
+ sanitize_patterns(data_files),
+ base_path=base_path,
+ download_config=DownloadConfig(token=token, storage_options=self.storage_options),
+ )
+
+ # Prepare config: DatasetConfig contains name, version and description but can be extended by each dataset
+ if "features" in inspect.signature(self.BUILDER_CONFIG_CLASS.__init__).parameters and features is not None:
+ config_kwargs["features"] = features
+ if data_files is not None:
+ config_kwargs["data_files"] = data_files
+ if data_dir is not None:
+ config_kwargs["data_dir"] = data_dir
+ self.config_kwargs = config_kwargs
+ self.config, self.config_id = self._create_builder_config(
+ config_name=config_name,
+ custom_features=features,
+ config_id=config_id,
+ **config_kwargs,
+ )
+
+ # prepare info: DatasetInfo are a standardized dataclass across all datasets
+ # Prefill datasetinfo
+ if info is None:
+ info = self._info()
+ info.builder_name = self.name
+ info.dataset_name = self.dataset_name
+ info.config_name = self.config.name
+ info.version = self.config.version
+ self.info = info
+ # update info with user specified infos
+ if features is not None:
+ self.info.features = features
+
+ # Prepare data dirs:
+ # cache_dir can be a remote bucket on GCS or S3
+ self._cache_dir_root = str(cache_dir or config.HF_DATASETS_CACHE)
+ self._cache_dir_root = (
+ self._cache_dir_root if is_remote_url(self._cache_dir_root) else os.path.expanduser(self._cache_dir_root)
+ )
+ self._cache_downloaded_dir = (
+ posixpath.join(self._cache_dir_root, config.DOWNLOADED_DATASETS_DIR)
+ if cache_dir
+ else str(config.DOWNLOADED_DATASETS_PATH)
+ )
+ self._cache_downloaded_dir = (
+ self._cache_downloaded_dir
+ if is_remote_url(self._cache_downloaded_dir)
+ else os.path.expanduser(self._cache_downloaded_dir)
+ )
+
+ # In case there exists a legacy cache directory
+ self._legacy_relative_data_dir = None
+
+ self._cache_dir = self._build_cache_dir()
+ if not is_remote_url(self._cache_dir_root):
+ os.makedirs(self._cache_dir_root, exist_ok=True)
+ lock_path = os.path.join(
+ self._cache_dir_root, Path(self._cache_dir).as_posix().replace("/", "_") + ".lock"
+ )
+ with FileLock(lock_path):
+ if os.path.exists(self._cache_dir): # check if data exist
+ if len(os.listdir(self._cache_dir)) > 0:
+ if os.path.exists(os.path.join(self._cache_dir, config.DATASET_INFO_FILENAME)):
+ logger.debug("Overwrite dataset info from restored data version if exists.")
+ self.info = DatasetInfo.from_directory(self._cache_dir)
+ else: # dir exists but no data, remove the empty dir as data aren't available anymore
+ logger.warning(
+ f"Old caching folder {self._cache_dir} for dataset {self.dataset_name} exists but no data were found. Removing it. "
+ )
+ os.rmdir(self._cache_dir)
+
+ # Store in the cache by default unless the user specifies a custom output_dir to download_and_prepare
+ self._output_dir = self._cache_dir
+ self._fs: fsspec.AbstractFileSystem = fsspec.filesystem("file")
+
+ # Set download manager
+ self.dl_manager = None
+
+ # Set to True by "datasets-cli test" to generate file checksums for (deprecated) dataset_infos.json independently of verification_mode value.
+ self._record_checksums = False
+
+ # Set in `.download_and_prepare` once the format of the generated dataset is known
+ self._file_format = None
+
+ # Enable streaming (e.g. it patches "open" to work with remote files)
+ extend_dataset_builder_for_streaming(self)
+
+ def __getstate__(self):
+ return self.__dict__
+
+ def __setstate__(self, d):
+ self.__dict__ = d
+ # Re-enable streaming, since patched functions are not kept when pickling
+ extend_dataset_builder_for_streaming(self)
+
+ def _check_legacy_cache(self) -> Optional[str]:
+ """Check for the old cache directory template {cache_dir}/{namespace}___{builder_name} from 2.13"""
+ if (
+ self.__module__.startswith("datasets.")
+ and not is_remote_url(self._cache_dir_root)
+ and self.config.name == "default"
+ ):
+ from .packaged_modules import _PACKAGED_DATASETS_MODULES
+
+ namespace = self.repo_id.split("/")[0] if self.repo_id and self.repo_id.count("/") > 0 else None
+ config_name = self.repo_id.replace("/", "--") if self.repo_id is not None else self.dataset_name
+ config_id = config_name + self.config_id[len(self.config.name) :]
+ hash = _PACKAGED_DATASETS_MODULES.get(self.name, "missing")[1]
+ legacy_relative_data_dir = posixpath.join(
+ self.dataset_name if namespace is None else f"{namespace}___{self.dataset_name}",
+ config_id,
+ "0.0.0",
+ hash,
+ )
+ legacy_cache_dir = posixpath.join(self._cache_dir_root, legacy_relative_data_dir)
+ if os.path.isdir(legacy_cache_dir):
+ return legacy_relative_data_dir
+
+ def _check_legacy_cache2(self, dataset_module: "DatasetModule") -> Optional[str]:
+ """Check for the old cache directory template {cache_dir}/{namespace}___{dataset_name}/{config_name}-xxx from 2.14 and 2.15"""
+ if (
+ self.__module__.startswith("datasets.")
+ and not is_remote_url(self._cache_dir_root)
+ and not (set(self.config_kwargs) - {"data_files", "data_dir"})
+ ):
+ from .packaged_modules import _PACKAGED_DATASETS_MODULES_2_15_HASHES
+ from .utils._dill import Pickler
+
+ def update_hash_with_config_parameters(hash: str, config_parameters: dict) -> str:
+ """
+ Used to update hash of packaged modules which is used for creating unique cache directories to reflect
+ different config parameters which are passed in metadata from readme.
+ """
+ params_to_exclude = {"config_name", "version", "description"}
+ params_to_add_to_hash = {
+ param: value
+ for param, value in sorted(config_parameters.items())
+ if param not in params_to_exclude
+ }
+ m = Hasher()
+ m.update(hash)
+ m.update(params_to_add_to_hash)
+ return m.hexdigest()
+
+ namespace = self.repo_id.split("/")[0] if self.repo_id and self.repo_id.count("/") > 0 else None
+ with patch.object(Pickler, "_legacy_no_dict_keys_sorting", True):
+ config_id = self.config.name + "-" + Hasher.hash({"data_files": self.config.data_files})
+ hash = _PACKAGED_DATASETS_MODULES_2_15_HASHES.get(self.name, "missing")
+ if (
+ dataset_module.builder_configs_parameters.metadata_configs
+ and self.config.name in dataset_module.builder_configs_parameters.metadata_configs
+ ):
+ hash = update_hash_with_config_parameters(
+ hash, dataset_module.builder_configs_parameters.metadata_configs[self.config.name]
+ )
+ legacy_relative_data_dir = posixpath.join(
+ self.dataset_name if namespace is None else f"{namespace}___{self.dataset_name}",
+ config_id,
+ "0.0.0",
+ hash,
+ )
+ legacy_cache_dir = posixpath.join(self._cache_dir_root, legacy_relative_data_dir)
+ if os.path.isdir(legacy_cache_dir):
+ return legacy_relative_data_dir
+
+ def _create_builder_config(
+ self, config_name=None, custom_features=None, config_id=None, **config_kwargs
+ ) -> tuple[BuilderConfig, str]:
+ """Create and validate BuilderConfig object as well as a unique config id for this config.
+ Raises ValueError if there are multiple builder configs and config_name and DEFAULT_CONFIG_NAME are None.
+ config_kwargs override the defaults kwargs in config
+ """
+ builder_config = None
+
+ # try default config
+ if config_name is None and self.BUILDER_CONFIGS:
+ if self.DEFAULT_CONFIG_NAME is not None:
+ builder_config = self.builder_configs.get(self.DEFAULT_CONFIG_NAME)
+ logger.info(f"No config specified, defaulting to: {self.dataset_name}/{builder_config.name}")
+ else:
+ if len(self.BUILDER_CONFIGS) > 1:
+ if not config_kwargs:
+ example_of_usage = (
+ f"load_dataset('{self.repo_id or self.dataset_name}', '{self.BUILDER_CONFIGS[0].name}')"
+ )
+ raise ValueError(
+ "Config name is missing."
+ f"\nPlease pick one among the available configs: {list(self.builder_configs.keys())}"
+ + f"\nExample of usage:\n\t`{example_of_usage}`"
+ )
+ else:
+ builder_config = self.BUILDER_CONFIGS[0]
+ logger.info(
+ f"No config specified, defaulting to the single config: {self.dataset_name}/{builder_config.name}"
+ )
+
+ # try to get config by name
+ if isinstance(config_name, str):
+ builder_config = self.builder_configs.get(config_name)
+ if builder_config is None and self.BUILDER_CONFIGS:
+ raise ValueError(
+ f"BuilderConfig '{config_name}' not found. Available: {list(self.builder_configs.keys())}"
+ )
+
+ # if not using an existing config, then create a new config on the fly
+ if not builder_config:
+ if config_name is not None:
+ config_kwargs["name"] = config_name
+ elif self.DEFAULT_CONFIG_NAME and not config_kwargs:
+ # Use DEFAULT_CONFIG_NAME only if no config_kwargs are passed
+ config_kwargs["name"] = self.DEFAULT_CONFIG_NAME
+ if "version" not in config_kwargs and hasattr(self, "VERSION") and self.VERSION:
+ config_kwargs["version"] = self.VERSION
+ builder_config = self.BUILDER_CONFIG_CLASS(**config_kwargs)
+
+ # otherwise use the config_kwargs to overwrite the attributes
+ else:
+ builder_config = copy.deepcopy(builder_config) if config_kwargs else builder_config
+ for key, value in config_kwargs.items():
+ if value is not None:
+ if not hasattr(builder_config, key):
+ raise ValueError(f"BuilderConfig {builder_config} doesn't have a '{key}' key.")
+ setattr(builder_config, key, value)
+
+ if not builder_config.name:
+ raise ValueError(f"BuilderConfig must have a name, got {builder_config.name}")
+
+ # resolve data files if needed
+ builder_config._resolve_data_files(
+ base_path=self.base_path,
+ download_config=DownloadConfig(token=self.token, storage_options=self.storage_options),
+ )
+
+ # compute the config id that is going to be used for caching
+ if config_id is None:
+ config_id = builder_config.create_config_id(
+ config_kwargs,
+ custom_features=custom_features,
+ )
+ is_custom = (config_id not in self.builder_configs) and config_id != "default"
+ if is_custom:
+ logger.info(f"Using custom data configuration {config_id}")
+ else:
+ if (
+ builder_config.name in self.builder_configs
+ and builder_config != self.builder_configs[builder_config.name]
+ ):
+ raise ValueError(
+ "Cannot name a custom BuilderConfig the same as an available "
+ f"BuilderConfig. Change the name. Available BuilderConfigs: {list(self.builder_configs.keys())}"
+ )
+ if not builder_config.version:
+ raise ValueError(f"BuilderConfig {builder_config.name} must have a version")
+
+ return builder_config, config_id
+
+ @classproperty
+ @classmethod
+ @memoize()
+ def builder_configs(cls) -> dict[str, BuilderConfig]:
+ """Dictionary of pre-defined configurations for this builder class."""
+ configs = {config.name: config for config in cls.BUILDER_CONFIGS}
+ if len(configs) != len(cls.BUILDER_CONFIGS):
+ names = [config.name for config in cls.BUILDER_CONFIGS]
+ raise ValueError(f"Names in BUILDER_CONFIGS must not be duplicated. Got {names}")
+ return configs
+
+ @property
+ def cache_dir(self):
+ return self._cache_dir
+
+ def _use_legacy_cache_dir_if_possible(self, dataset_module: "DatasetModule"):
+ # Check for the legacy cache directory template (datasets<3.0.0)
+ self._legacy_relative_data_dir = (
+ self._check_legacy_cache2(dataset_module) or self._check_legacy_cache() or None
+ )
+ self._cache_dir = self._build_cache_dir()
+ self._output_dir = self._cache_dir
+
+ def _relative_data_dir(self, with_version=True, with_hash=True) -> str:
+ """Relative path of this dataset in cache_dir:
+ Will be:
+ self.dataset_name/self.config.version/self.hash/
+ or if a repo_id with a namespace has been specified:
+ self.namespace___self.dataset_name/self.config.version/self.hash/
+ If any of these element is missing or if ``with_version=False`` the corresponding subfolders are dropped.
+ """
+ if self._legacy_relative_data_dir is not None and with_version and with_hash:
+ return self._legacy_relative_data_dir
+
+ namespace = self.repo_id.split("/")[0] if self.repo_id and self.repo_id.count("/") > 0 else None
+ builder_data_dir = self.dataset_name if namespace is None else f"{namespace}___{self.dataset_name}"
+ builder_data_dir = posixpath.join(builder_data_dir, self.config_id)
+ if with_version:
+ builder_data_dir = posixpath.join(builder_data_dir, str(self.config.version))
+ if with_hash and self.hash and isinstance(self.hash, str):
+ builder_data_dir = posixpath.join(builder_data_dir, self.hash)
+ return builder_data_dir
+
+ def _build_cache_dir(self):
+ """Return the data directory for the current version."""
+ builder_data_dir = posixpath.join(self._cache_dir_root, self._relative_data_dir(with_version=False))
+ version_data_dir = posixpath.join(self._cache_dir_root, self._relative_data_dir(with_version=True))
+
+ def _other_versions_on_disk():
+ """Returns previous versions on disk."""
+ if not os.path.exists(builder_data_dir):
+ return []
+
+ version_dirnames = []
+ for dir_name in os.listdir(builder_data_dir):
+ try:
+ version_dirnames.append((utils.Version(dir_name), dir_name))
+ except ValueError: # Invalid version (ex: incomplete data dir)
+ pass
+ version_dirnames.sort(reverse=True)
+ return version_dirnames
+
+ # Check and warn if other versions exist
+ if not is_remote_url(builder_data_dir):
+ version_dirs = _other_versions_on_disk()
+ if version_dirs:
+ other_version = version_dirs[0][0]
+ if other_version != self.config.version:
+ warn_msg = (
+ f"Found a different version {str(other_version)} of dataset {self.dataset_name} in "
+ f"cache_dir {self._cache_dir_root}. Using currently defined version "
+ f"{str(self.config.version)}."
+ )
+ logger.warning(warn_msg)
+
+ return version_data_dir
+
+ @abc.abstractmethod
+ def _info(self) -> DatasetInfo:
+ """Construct the DatasetInfo object. See `DatasetInfo` for details.
+
+ Warning: This function is only called once and the result is cached for all
+ following .info() calls.
+
+ Returns:
+ info: (DatasetInfo) The dataset information
+ """
+ raise NotImplementedError
+
+ @classmethod
+ def get_imported_module_dir(cls):
+ """Return the path of the module of this class or subclass."""
+ return os.path.dirname(inspect.getfile(inspect.getmodule(cls)))
+
+ def _rename(self, src: str, dst: str):
+ rename(self._fs, src, dst)
+
+ def download_and_prepare(
+ self,
+ output_dir: Optional[str] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ verification_mode: Optional[Union[VerificationMode, str]] = None,
+ dl_manager: Optional[DownloadManager] = None,
+ base_path: Optional[str] = None,
+ file_format: str = "arrow",
+ max_shard_size: Optional[Union[int, str]] = None,
+ num_proc: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ **download_and_prepare_kwargs,
+ ):
+ """Downloads and prepares dataset for reading.
+
+ Args:
+ output_dir (`str`, *optional*):
+ Output directory for the dataset.
+ Default to this builder's `cache_dir`, which is inside `~/.cache/huggingface/datasets` by default.
+
+
+ download_config (`DownloadConfig`, *optional*):
+ Specific download configuration parameters.
+ download_mode ([`DownloadMode`] or `str`, *optional*):
+ Select the download/generate mode, default to `REUSE_DATASET_IF_EXISTS`.
+ verification_mode ([`VerificationMode`] or `str`, defaults to `BASIC_CHECKS`):
+ Verification mode determining the checks to run on the downloaded/processed dataset information (checksums/size/splits/...).
+
+
+ dl_manager (`DownloadManager`, *optional*):
+ Specific `DownloadManger` to use.
+ base_path (`str`, *optional*):
+ Base path for relative paths that are used to download files. This can be a remote url.
+ If not specified, the value of the `base_path` attribute (`self.base_path`) will be used instead.
+ file_format (`str`, *optional*):
+ Format of the data files in which the dataset will be written.
+ Supported formats: "arrow", "parquet". Default to "arrow" format.
+ If the format is "parquet", then image and audio data are embedded into the Parquet files instead of pointing to local files.
+
+
+ max_shard_size (`Union[str, int]`, *optional*):
+ Maximum number of bytes written per shard, default is "500MB".
+ The size is based on uncompressed data size, so in practice your shard files may be smaller than
+ `max_shard_size` thanks to Parquet compression for example.
+
+
+ num_proc (`int`, *optional*, defaults to `None`):
+ Number of processes when downloading and generating the dataset locally.
+ Multiprocessing is disabled by default.
+
+
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the caching file-system backend, if any.
+
+
+ **download_and_prepare_kwargs (additional keyword arguments): Keyword arguments.
+
+ Example:
+
+ Download and prepare the dataset as Arrow files that can be loaded as a Dataset using `builder.as_dataset()`:
+
+ ```py
+ >>> from datasets import load_dataset_builder
+ >>> builder = load_dataset_builder("cornell-movie-review-data/rotten_tomatoes")
+ >>> builder.download_and_prepare()
+ ```
+
+ Download and prepare the dataset as sharded Parquet files locally:
+
+ ```py
+ >>> from datasets import load_dataset_builder
+ >>> builder = load_dataset_builder("cornell-movie-review-data/rotten_tomatoes")
+ >>> builder.download_and_prepare("./output_dir", file_format="parquet")
+ ```
+
+ Download and prepare the dataset as sharded Parquet files in a cloud storage:
+
+ ```py
+ >>> from datasets import load_dataset_builder
+ >>> storage_options = {"key": aws_access_key_id, "secret": aws_secret_access_key}
+ >>> builder = load_dataset_builder("cornell-movie-review-data/rotten_tomatoes")
+ >>> builder.download_and_prepare("s3://my-bucket/my_rotten_tomatoes", storage_options=storage_options, file_format="parquet")
+ ```
+ """
+ output_dir = output_dir if output_dir is not None else self._cache_dir
+ # output_dir can be a remote bucket on GCS or S3
+ fs, output_dir = url_to_fs(output_dir, **(storage_options or {}))
+ self._fs = fs
+ self._output_dir = output_dir if not is_remote_filesystem(self._fs) else self._fs.unstrip_protocol(output_dir)
+
+ download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS)
+ verification_mode = VerificationMode(verification_mode or VerificationMode.BASIC_CHECKS)
+ base_path = base_path if base_path is not None else self.base_path
+
+ if file_format is not None and file_format not in ["arrow", "parquet"]:
+ raise ValueError(f"Unsupported file_format: {file_format}. Expected 'arrow' or 'parquet'")
+ self._file_format = file_format
+
+ if self._fs._strip_protocol(self._output_dir) == "":
+ # We don't support the root directory, because it has no dirname,
+ # and we need a dirname to use a .incomplete directory
+ # when the dataset is being written
+ raise RuntimeError(
+ f"Unable to download and prepare the dataset at the root {self._output_dir}. "
+ f"Please specify a subdirectory, e.g. '{self._output_dir + self.dataset_name}'"
+ )
+
+ if dl_manager is None:
+ if download_config is None:
+ download_config = DownloadConfig(
+ cache_dir=self._cache_downloaded_dir,
+ force_download=download_mode == DownloadMode.FORCE_REDOWNLOAD,
+ force_extract=download_mode == DownloadMode.FORCE_REDOWNLOAD,
+ use_etag=False,
+ num_proc=num_proc,
+ token=self.token,
+ storage_options=self.storage_options,
+ ) # We don't use etag for data files to speed up the process
+
+ dl_manager = DownloadManager(
+ dataset_name=self.dataset_name,
+ download_config=download_config,
+ data_dir=self.config.data_dir,
+ base_path=base_path,
+ record_checksums=self._record_checksums,
+ )
+
+ is_local = not is_remote_filesystem(self._fs)
+ self.dl_manager = dl_manager
+
+ # Prevent parallel local disk operations
+ if is_local:
+ # Create parent directory of the output_dir to put the lock file in there
+ Path(self._output_dir).parent.mkdir(parents=True, exist_ok=True)
+ lock_path = self._output_dir + "_builder.lock"
+
+ # File locking only with local paths; no file locking on GCS or S3
+ with FileLock(lock_path) if is_local else contextlib.nullcontext():
+ # Check if the data already exists
+ data_exists = self._fs.exists(posixpath.join(self._output_dir, config.DATASET_INFO_FILENAME))
+ if data_exists and download_mode == DownloadMode.REUSE_DATASET_IF_EXISTS:
+ logger.info(f"Found cached dataset {self.dataset_name} ({self._output_dir})")
+ # We need to update the info in case some splits were added in the meantime
+ # for example when calling load_dataset from multiple workers.
+ self.info = self._load_info()
+ return
+
+ logger.info(f"Generating dataset {self.dataset_name} ({self._output_dir})")
+ if is_local: # if cache dir is local, check for available space
+ if not has_sufficient_disk_space(
+ self.info.size_in_bytes or 0, directory=Path(self._output_dir).parent
+ ):
+ raise OSError(
+ f"Not enough disk space. Needed: {size_str(self.info.size_in_bytes or 0)} (download: {size_str(self.info.download_size or 0)}, generated: {size_str(self.info.dataset_size or 0)}"
+ )
+
+ @contextlib.contextmanager
+ def incomplete_dir(dirname):
+ """Create temporary dir for dirname and rename on exit."""
+ if not is_local:
+ self._fs.makedirs(dirname, exist_ok=True)
+ yield dirname
+ else:
+ tmp_dir = dirname + ".incomplete"
+ os.makedirs(tmp_dir, exist_ok=True)
+ try:
+ yield tmp_dir
+ if os.path.isdir(dirname):
+ shutil.rmtree(dirname)
+ # LocalFileSystem.mv does copy + rm, it is more efficient to simply rename a local directory
+ shutil.move(tmp_dir, dirname)
+ finally:
+ if os.path.exists(tmp_dir):
+ shutil.rmtree(tmp_dir)
+
+ # Print is intentional: we want this to always go to stdout so user has
+ # information needed to cancel download/preparation if needed.
+ # This comes right before the progress bar.
+ if self.info.size_in_bytes:
+ logger.info(
+ f"Downloading and preparing dataset {self.dataset_name}/{self.config.name} "
+ f"(download: {size_str(self.info.download_size)}, generated: {size_str(self.info.dataset_size)}, "
+ f"total: {size_str(self.info.size_in_bytes)}) to {self._output_dir}..."
+ )
+ else:
+ _dest = self._fs._strip_protocol(self._output_dir) if is_local else self._output_dir
+ logger.info(f"Downloading and preparing dataset {self.dataset_name}/{self.config.name} to {_dest}...")
+
+ # Create a tmp dir and rename to self._output_dir on successful exit.
+ with incomplete_dir(self._output_dir) as tmp_output_dir:
+ # Temporarily assign _output_dir to tmp_data_dir to avoid having to forward
+ # it to every sub function.
+ with temporary_assignment(self, "_output_dir", tmp_output_dir):
+ prepare_split_kwargs = {"file_format": file_format}
+ if max_shard_size is not None:
+ prepare_split_kwargs["max_shard_size"] = max_shard_size
+ if num_proc is not None:
+ prepare_split_kwargs["num_proc"] = num_proc
+ self._download_and_prepare(
+ dl_manager=dl_manager,
+ verification_mode=verification_mode,
+ **prepare_split_kwargs,
+ **download_and_prepare_kwargs,
+ )
+ # Sync info
+ self.info.dataset_size = sum(split.num_bytes for split in self.info.splits.values())
+ if dl_manager.record_checksums:
+ self.info.download_checksums = dl_manager.get_recorded_sizes_checksums()
+ if self.info.download_size is not None:
+ self.info.size_in_bytes = self.info.dataset_size + self.info.download_size
+ # Save info
+ self._save_info()
+
+ logger.info(
+ f"Dataset {self.dataset_name} downloaded and prepared to {self._output_dir}. "
+ f"Subsequent calls will reuse this data."
+ )
+
+ def _download_and_prepare(self, dl_manager, verification_mode, **prepare_split_kwargs):
+ """Downloads and prepares dataset for reading.
+
+ This is the internal implementation to overwrite called when user calls
+ `download_and_prepare`. It should download all required data and generate
+ the pre-processed datasets files.
+
+ Args:
+ dl_manager ([`DownloadManager`]):
+ `DownloadManager` used to download and cache data.
+ verification_mode ([`VerificationMode`]):
+ if `ALL_CHECKS`, perform all the verifications including checksums.
+ if `BASIC_CHECKS`, do not perform checksums, only perform split tests.
+ if `NO_CHECKS`, do not perform any verification.
+ prepare_split_kwargs: Additional options, such as `file_format`, `max_shard_size`
+ """
+ # Generating data for all splits
+ split_dict = SplitDict(dataset_name=self.dataset_name)
+ split_generators_kwargs = self._make_split_generators_kwargs(prepare_split_kwargs)
+ split_generators: list[SplitGenerator] = self._split_generators(dl_manager, **split_generators_kwargs)
+
+ # Checksums verification
+ if verification_mode == VerificationMode.ALL_CHECKS and dl_manager.record_checksums:
+ verify_checksums(
+ self.info.download_checksums, dl_manager.get_recorded_sizes_checksums(), "dataset source files"
+ )
+
+ # Build splits
+ for split_generator in split_generators:
+ if str(split_generator.split_info.name).lower() == "all":
+ raise ValueError(
+ "`all` is a special split keyword corresponding to the "
+ "union of all splits, so cannot be used as key in "
+ "._split_generator()."
+ )
+
+ logger.info(f"Generating {split_generator.split_info.name} split")
+ split_dict.add(split_generator.split_info)
+
+ try:
+ # Prepare split will record examples associated to the split
+ self._prepare_split(split_generator, **prepare_split_kwargs)
+ except OSError as e:
+ raise OSError("Cannot find data file. " + "\nOriginal error:\n" + str(e)) from None
+ dl_manager.manage_extracted_files()
+
+ if verification_mode == VerificationMode.BASIC_CHECKS or verification_mode == VerificationMode.ALL_CHECKS:
+ verify_splits(self.info.splits, split_dict)
+
+ # Update the info object with the splits.
+ self.info.splits = split_dict
+ self.info.download_size = dl_manager.downloaded_size
+
+ def _load_info(self) -> DatasetInfo:
+ return DatasetInfo.from_directory(self._output_dir, storage_options=self._fs.storage_options)
+
+ def _save_info(self):
+ file_lock = (
+ FileLock(self._output_dir + "_info.lock")
+ if not is_remote_filesystem(self._fs)
+ else contextlib.nullcontext()
+ )
+ with file_lock:
+ self.info.write_to_directory(self._output_dir, storage_options=self._fs.storage_options)
+
+ def _make_split_generators_kwargs(self, prepare_split_kwargs):
+ """Get kwargs for `self._split_generators()` from `prepare_split_kwargs`."""
+ del prepare_split_kwargs
+ return {}
+
+ def as_dataset(
+ self,
+ split: Optional[Union[str, Split, list[str], list[Split]]] = None,
+ in_memory=False,
+ ) -> Union[Dataset, DatasetDict]:
+ """Return a Dataset for the specified split.
+
+ Args:
+ split (`datasets.Split`):
+ Which subset of the data to return.
+ verification_mode ([`VerificationMode`] or `str`, defaults to `BASIC_CHECKS`):
+ Verification mode determining the checks to run on the
+ downloaded/processed dataset information (checksums/size/splits/...).
+
+
+ in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+
+ Returns:
+ datasets.Dataset
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset_builder
+ >>> builder = load_dataset_builder('cornell-movie-review-data/rotten_tomatoes')
+ >>> builder.download_and_prepare()
+ >>> ds = builder.as_dataset(split='train')
+ >>> ds
+ Dataset({
+ features: ['text', 'label'],
+ num_rows: 8530
+ })
+ ```
+ """
+ if self._file_format is not None and self._file_format != "arrow":
+ raise FileFormatError('Loading a dataset not written in the "arrow" format is not supported.')
+ if is_remote_filesystem(self._fs):
+ raise NotImplementedError(f"Loading a dataset cached in a {type(self._fs).__name__} is not supported.")
+ if not os.path.exists(self._output_dir):
+ raise FileNotFoundError(
+ f"Dataset {self.dataset_name}: could not find data in {self._output_dir}. Please make sure to call "
+ "builder.download_and_prepare(), or use "
+ "datasets.load_dataset() before trying to access the Dataset object."
+ )
+
+ logger.debug(f"Constructing Dataset for split {split or ', '.join(self.info.splits)}, from {self._output_dir}")
+
+ # By default, return all splits
+ if split is None:
+ split = {s: s for s in self.info.splits}
+
+ # Create a dataset for each of the given splits
+ datasets = map_nested(
+ partial(
+ self._build_single_dataset,
+ in_memory=in_memory,
+ ),
+ split,
+ map_tuple=True,
+ disable_tqdm=True,
+ )
+ if isinstance(datasets, dict):
+ datasets = DatasetDict(datasets)
+ return datasets
+
+ def _build_single_dataset(
+ self,
+ split: Union[str, ReadInstruction, Split],
+ in_memory: bool = False,
+ ):
+ """as_dataset for a single split."""
+ if not isinstance(split, ReadInstruction):
+ split = str(split)
+ if split == "all":
+ split = "+".join(self.info.splits.keys())
+ split = Split(split)
+
+ # Build base dataset
+ ds = self._as_dataset(
+ split=split,
+ in_memory=in_memory,
+ )
+ return ds
+
+ def _as_dataset(self, split: Union[ReadInstruction, Split] = Split.TRAIN, in_memory: bool = False) -> Dataset:
+ """Constructs a `Dataset`.
+
+ This is the internal implementation to overwrite called when user calls
+ `as_dataset`. It should read the pre-processed datasets files and generate
+ the `Dataset` object.
+
+ Args:
+ split (`datasets.Split`):
+ which subset of the data to read.
+ in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+
+ Returns:
+ `Dataset`
+ """
+ cache_dir = self._fs._strip_protocol(self._output_dir)
+ dataset_name = self.dataset_name
+ if self._check_legacy_cache():
+ dataset_name = self.name
+ dataset_kwargs = ArrowReader(cache_dir, self.info).read(
+ name=dataset_name,
+ instructions=split,
+ split_infos=self.info.splits.values(),
+ in_memory=in_memory,
+ )
+ fingerprint = self._get_dataset_fingerprint(split)
+ return Dataset(fingerprint=fingerprint, **dataset_kwargs)
+
+ def _get_dataset_fingerprint(self, split: Union[ReadInstruction, Split]) -> str:
+ """The dataset fingerprint is the hash of the relative directory dataset_name/config_name/version/hash, as well as the split specs."""
+ hasher = Hasher()
+ hasher.update(Path(self._relative_data_dir()).as_posix())
+ hasher.update(str(split)) # for example: train, train+test, train[:10%], test[:33%](pct1_dropremainder)
+ fingerprint = hasher.hexdigest()
+ return fingerprint
+
+ def as_streaming_dataset(
+ self,
+ split: Optional[str] = None,
+ base_path: Optional[str] = None,
+ ) -> Union[dict[str, IterableDataset], IterableDataset]:
+ if is_remote_filesystem(self._fs):
+ raise NotImplementedError(
+ f"Loading a streaming dataset cached in a {type(self._fs).__name__} is not supported yet."
+ )
+
+ dl_manager = StreamingDownloadManager(
+ base_path=base_path or self.base_path,
+ download_config=DownloadConfig(token=self.token, storage_options=self.storage_options),
+ dataset_name=self.dataset_name,
+ data_dir=self.config.data_dir,
+ )
+ splits_generators = {sg.name: sg for sg in self._split_generators(dl_manager)}
+ # By default, return all splits
+ if split is None:
+ splits_generator = splits_generators
+ elif str(split) in splits_generators:
+ splits_generator = splits_generators[str(split)]
+ elif hasattr(split, "get_read_instruction"):
+ split_dict = SplitDict({name: sg.split_info for name, sg in splits_generators.items()})
+ split_instruction = split.get_read_instruction(split_dict)
+ split_infos = split_instruction.get_list_sliced_split_info()
+ if any(split_info.slice_value is not None for split_info in split_infos):
+ raise ValueError(f"Bad split: {split}. Available splits: {list(splits_generators)}")
+ return _concatenate_iterable_datasets(
+ [
+ self._as_streaming_dataset_single(splits_generators[split_info.split_info.name])
+ for split_info in split_infos
+ ],
+ info=self.info,
+ split=split,
+ )
+ elif str(split) == str(Split.ALL):
+ return _concatenate_iterable_datasets(
+ [self._as_streaming_dataset_single(sg) for sg in splits_generators.values()],
+ info=self.info,
+ split=split,
+ )
+ elif "+" in str(split) and "[" not in str(split) and "]" not in str(split):
+ split_names = [split_name.strip() for split_name in str(split).split("+")]
+ if not all(split_name in splits_generators for split_name in split_names):
+ raise ValueError(f"Bad split: {split}. Available splits: {list(splits_generators)}")
+ return _concatenate_iterable_datasets(
+ [self._as_streaming_dataset_single(splits_generators[split_name]) for split_name in split_names],
+ info=self.info,
+ split=split,
+ )
+ else:
+ raise ValueError(f"Bad split: {split}. Available splits: {list(splits_generators)}")
+
+ # Create a dataset for each of the given splits
+ datasets = map_nested(
+ self._as_streaming_dataset_single,
+ splits_generator,
+ map_tuple=True,
+ )
+ if isinstance(datasets, dict):
+ datasets = IterableDatasetDict(datasets)
+ return datasets
+
+ def _as_streaming_dataset_single(
+ self,
+ splits_generator,
+ ) -> IterableDataset:
+ ex_iterable = self._get_examples_iterable_for_split(splits_generator)
+ # add auth to be able to access and decode audio/image files from private repositories.
+ token_per_repo_id = {self.repo_id: self.token} if self.repo_id else {}
+ return IterableDataset(
+ ex_iterable, info=self.info, split=splits_generator.name, token_per_repo_id=token_per_repo_id
+ )
+
+ @abc.abstractmethod
+ def _split_generators(self, dl_manager: Union[DownloadManager, StreamingDownloadManager]):
+ """Specify feature dictionary generators and dataset splits.
+
+ This function returns a list of `SplitGenerator`s defining how to generate
+ data and what splits to use.
+
+ Example:
+
+ return [
+ datasets.SplitGenerator(
+ name=datasets.Split.TRAIN,
+ gen_kwargs={'file': 'train_data.zip'},
+ ),
+ datasets.SplitGenerator(
+ name=datasets.Split.TEST,
+ gen_kwargs={'file': 'test_data.zip'},
+ ),
+ ]
+
+ The above code will first call `_generate_examples(file='train_data.zip')`
+ to write the train data, then `_generate_examples(file='test_data.zip')` to
+ write the test data.
+
+ Datasets are typically split into different subsets to be used at various
+ stages of training and evaluation.
+
+ Note that for datasets without a `VALIDATION` split, you can use a
+ fraction of the `TRAIN` data for evaluation as you iterate on your model
+ so as not to overfit to the `TEST` data.
+
+ For downloads and extractions, use the given `download_manager`.
+ Note that the `DownloadManager` caches downloads, so it is fine to have each
+ generator attempt to download the source data.
+
+ A good practice is to download all data in this function, and then
+ distribute the relevant parts to each split with the `gen_kwargs` argument
+
+ Args:
+ dl_manager (`Union[DownloadManager, StreamingDownloadManager]`):
+ Download manager to download the data
+
+ Returns:
+ `list`.
+ """
+ raise NotImplementedError()
+
+ @abc.abstractmethod
+ def _prepare_split(
+ self,
+ split_generator: SplitGenerator,
+ file_format: str = "arrow",
+ max_shard_size: Optional[Union[str, int]] = None,
+ num_proc: Optional[int] = None,
+ **kwargs,
+ ):
+ """Generate the examples and record them on disk.
+
+ Args:
+ split_generator (`SplitGenerator`):
+ Split generator to process
+ file_format (`str`, *optional*):
+ format of the data files in which the dataset will be written.
+ Supported formats: "arrow", "parquet". Default to "arrow" format.
+ max_shard_size (`Union[str, int]`, *optional*):
+ Maximum number of bytes written per shard, default is "500MB".
+ The size is based on uncompressed data size, so in practice your shard files may be smaller than
+ `max_shard_size` thanks to Parquet compression for example.
+ num_proc (`int`, *optional*, defaults to `None`):
+ Number of processes when downloading and generating the dataset locally.
+ Multiprocessing is disabled by default.
+
+
+ **kwargs: Additional kwargs forwarded from _download_and_prepare
+ """
+ raise NotImplementedError()
+
+ def _get_examples_iterable_for_split(self, split_generator: SplitGenerator) -> ExamplesIterable:
+ """Generate the examples on the fly.
+
+ Args:
+ split_generator (`SplitGenerator`):
+ Split generator to process
+ """
+ raise NotImplementedError()
+
+
+@dataclass
+class Key:
+ original_shard_id: int
+ item_or_batch_id: int
+
+ def __str__(self):
+ return str((self.original_shard_id, self.item_or_batch_id))
+
+
+class GeneratorBasedBuilder(DatasetBuilder):
+ """Base class for datasets with data generation based on dict generators.
+
+ `GeneratorBasedBuilder` is a convenience class that abstracts away much
+ of the data writing and reading of `DatasetBuilder`. It expects subclasses to
+ implement generators of feature dictionaries across the dataset splits
+ (`_split_generators`). See the method docstrings for details.
+ """
+
+ def _generate_shards(self, **kwargs) -> Iterator[Union[str, dict[str, Any]]]:
+ """Default function generating shards paths for each `SplitGenerator`.
+
+ This function is useful to list the original shards from where the data
+ comes from and is either converted to Arrow or streamed to an IterableDataset.
+
+ This is optional and only used for certain utilities, but not in Dataset
+ nor IterableDataset. E.g. it's used to map original shard files to Parquet
+ files in the Dataset Viewer after conversion.
+
+ Args:
+ **kwargs (additional keyword arguments):
+ Arguments forwarded from the SplitGenerator.gen_kwargs
+
+ Yields:
+ shard: generally a string representing the shard path, or a dict
+ representing the shard in case of shards spanning intra or inter-files.
+ """
+ raise NotImplementedError()
+
+ @abc.abstractmethod
+ def _generate_examples(self, **kwargs) -> Iterator[tuple[Key, dict[str, Any]]]:
+ """Default function generating examples for each `SplitGenerator`.
+
+ This function preprocess the examples from the raw data to the preprocessed
+ dataset files.
+ This function is called once for each `SplitGenerator` defined in
+ `_split_generators`. The examples yielded here will be written on
+ disk.
+
+ Args:
+ **kwargs (additional keyword arguments):
+ Arguments forwarded from the SplitGenerator.gen_kwargs
+
+ Yields:
+ key: `str` or `int`, a unique deterministic example identification key.
+ * Unique: An error will be raised if two examples are yield with the
+ same key.
+ * Deterministic: When generating the dataset twice, the same example
+ should have the same key.
+ Good keys can be the image id, or line number if examples are extracted
+ from a text file.
+ The key will be hashed and sorted to shuffle examples deterministically,
+ such as generating the dataset multiple times keep examples in the
+ same order.
+ example: `dict`, a feature dictionary
+ ready to be encoded and written to disk. The example will be
+ encoded with `self.info.features.encode_example({...})`.
+ """
+ raise NotImplementedError()
+
+ def _prepare_split(
+ self,
+ split_generator: SplitGenerator,
+ file_format="arrow",
+ num_proc: Optional[int] = None,
+ max_shard_size: Optional[Union[int, str]] = None,
+ ):
+ max_shard_size = convert_file_size_to_int(max_shard_size or config.MAX_SHARD_SIZE)
+
+ if self.info.splits is not None:
+ split_info = self.info.splits[split_generator.name]
+ else:
+ split_info = split_generator.split_info
+
+ SUFFIX = "-JJJJJ-SSSSS-of-NNNNN"
+ fname = f"{self.dataset_name}-{split_generator.name}{SUFFIX}.{file_format}"
+ fpath = posixpath.join(self._output_dir, fname)
+
+ if num_proc and num_proc > 1:
+ num_original_shards = _number_of_shards_in_gen_kwargs(split_generator.gen_kwargs)
+ if num_original_shards <= 1:
+ logger.warning(
+ f"Setting num_proc from {num_proc} back to 1 for the {split_info.name} split to disable multiprocessing as it only contains one shard."
+ )
+ num_proc = 1
+ elif num_original_shards < num_proc:
+ logger.warning(
+ f"Setting num_proc from {num_proc} to {num_original_shards} for the {split_info.name} split as it only contains {num_original_shards} shards."
+ )
+ num_proc = num_original_shards
+
+ pbar = hf_tqdm(
+ unit=" examples",
+ total=split_info.num_examples,
+ desc=f"Generating {split_info.name} split",
+ )
+
+ _prepare_split_args = {
+ "fpath": fpath,
+ "file_format": file_format,
+ "max_shard_size": max_shard_size,
+ "split_info": split_info,
+ }
+
+ if num_proc is None or num_proc == 1:
+ result = None
+ gen_kwargs = split_generator.gen_kwargs
+ job_id = 0
+ with pbar:
+ for job_id, done, content in self._prepare_split_single(
+ gen_kwargs=gen_kwargs, job_id=job_id, **_prepare_split_args
+ ):
+ if done:
+ result = content
+ else:
+ pbar.update(content)
+ # wrapping everything into lists for consistency with the multiprocessed code path
+ assert result is not None, "Failed to retrieve results from prepare_split"
+ (
+ examples_per_job,
+ bytes_per_job,
+ features_per_job,
+ shards_per_job,
+ shard_lengths_per_job,
+ original_shards_per_job,
+ original_shard_lengths_per_job,
+ ) = ([item] for item in result)
+ else:
+ kwargs_per_job = [
+ {"gen_kwargs": gen_kwargs, "job_id": job_id, **_prepare_split_args}
+ for job_id, gen_kwargs in enumerate(
+ _split_gen_kwargs(split_generator.gen_kwargs, max_num_jobs=num_proc)
+ )
+ ]
+ num_jobs = len(kwargs_per_job)
+
+ examples_per_job = [None] * num_jobs
+ bytes_per_job = [None] * num_jobs
+ features_per_job = [None] * num_jobs
+ shards_per_job = [None] * num_jobs
+ shard_lengths_per_job = [None] * num_jobs
+ original_shards_per_job = [None] * num_jobs
+ original_shard_lengths_per_job = [None] * num_jobs
+
+ with Pool(num_proc) as pool:
+ with pbar:
+ for job_id, done, content in iflatmap_unordered(
+ pool, self._prepare_split_single, kwargs_iterable=kwargs_per_job
+ ):
+ if done:
+ # the content is the result of the job
+ (
+ examples_per_job[job_id],
+ bytes_per_job[job_id],
+ features_per_job[job_id],
+ shards_per_job[job_id],
+ shard_lengths_per_job[job_id],
+ original_shards_per_job[job_id],
+ original_shard_lengths_per_job[job_id],
+ ) = content
+ else:
+ # the content is the number of examples progress update
+ pbar.update(content)
+
+ assert None not in examples_per_job, (
+ f"Failed to retrieve results from prepare_split: result list {examples_per_job} still contains None - at least one worker failed to return its results"
+ )
+
+ total_shards = sum(shards_per_job)
+ total_original_shards = sum(original_shards_per_job)
+ total_num_examples = sum(examples_per_job)
+ total_num_bytes = sum(bytes_per_job)
+ features = features_per_job[0]
+
+ split_generator.split_info.num_examples = total_num_examples
+ split_generator.split_info.num_bytes = total_num_bytes
+
+ # should rename everything at the end
+ logger.debug(f"Renaming {total_shards} shards.")
+ if total_shards > 1:
+ # use the -SSSSS-of-NNNNN pattern
+
+ def _rename_shard(shard_and_job: tuple[int]):
+ shard_id, job_id = shard_and_job
+ global_shard_id = sum(shards_per_job[:job_id]) + shard_id
+ self._rename(
+ fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
+ fpath.replace("JJJJJ-SSSSS", f"{global_shard_id:05d}").replace("NNNNN", f"{total_shards:05d}"),
+ )
+
+ shards_and_jobs = [
+ (shard_id, job_id)
+ for job_id, num_shards in enumerate(shards_per_job)
+ for shard_id in range(num_shards)
+ ]
+ thread_map(_rename_shard, shards_and_jobs, disable=True, max_workers=64)
+
+ split_generator.split_info.shard_lengths = [
+ shard_length for shard_lengths in shard_lengths_per_job for shard_length in shard_lengths
+ ]
+ else:
+ # don't use any pattern
+ shard_id, job_id = 0, 0
+ self._rename(
+ fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
+ fpath.replace(SUFFIX, ""),
+ )
+
+ if total_original_shards > 1 and config.SAVE_ORIGINAL_SHARD_LENGTHS:
+ split_generator.split_info.original_shard_lengths = [
+ original_shard_length
+ for original_shard_lengths in original_shard_lengths_per_job
+ for original_shard_length in original_shard_lengths
+ ]
+
+ if self.info.features is None:
+ self.info.features = features
+
+ def _prepare_split_single(
+ self,
+ gen_kwargs: dict,
+ fpath: str,
+ file_format: str,
+ max_shard_size: int,
+ split_info: SplitInfo,
+ job_id: int,
+ ) -> Iterator[tuple[int, bool, tuple[int, int, Features, int, int, int]]]:
+ generator = self._generate_examples(**gen_kwargs)
+ writer_class = ParquetWriter if file_format == "parquet" else ArrowWriter
+ embed_local_files = file_format == "parquet"
+ shard_lengths = []
+ original_shard_lengths = []
+ total_num_examples, total_num_bytes = 0, 0
+
+ shard_id = 0
+ original_shard_id = 0
+ num_examples_progress_update = 0
+ try:
+ writer = writer_class(
+ features=self.info.features,
+ path=fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
+ writer_batch_size=self._writer_batch_size,
+ storage_options=self._fs.storage_options,
+ embed_local_files=embed_local_files,
+ )
+ try:
+ _time = time.time()
+ for key, record in generator:
+ if isinstance(key, Key): # old custom builders may not use Key
+ original_shard_id = key.original_shard_id
+ if max_shard_size is not None and writer._num_bytes > max_shard_size:
+ num_examples, num_bytes = writer.finalize()
+ writer.close()
+ shard_lengths.append(num_examples)
+ total_num_examples += num_examples
+ total_num_bytes += num_bytes
+ shard_id += 1
+ writer = writer_class(
+ features=writer._features,
+ path=fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
+ writer_batch_size=self._writer_batch_size,
+ storage_options=self._fs.storage_options,
+ embed_local_files=embed_local_files,
+ )
+ example = self.info.features.encode_example(record) if self.info.features is not None else record
+ writer.write(example)
+ if len(original_shard_lengths) <= original_shard_id:
+ original_shard_lengths.extend([0] * (1 + original_shard_id - len(original_shard_lengths)))
+ original_shard_lengths[original_shard_id] += 1
+ num_examples_progress_update += 1
+ if time.time() > _time + config.PBAR_REFRESH_TIME_INTERVAL:
+ _time = time.time()
+ yield job_id, False, num_examples_progress_update
+ num_examples_progress_update = 0
+ finally:
+ yield job_id, False, num_examples_progress_update
+ num_shards = shard_id + 1
+ num_original_shards = original_shard_id + 1
+ num_examples, num_bytes = writer.finalize()
+ writer.close()
+ shard_lengths.append(num_examples)
+ total_num_examples += num_examples
+ total_num_bytes += num_bytes
+ except Exception as e:
+ # Ignore the writer's error for no examples written to the file if this error was caused by the error in _generate_examples before the first example was yielded
+ if isinstance(e, SchemaInferenceError) and e.__context__ is not None:
+ e = e.__context__
+ raise DatasetGenerationError("An error occurred while generating the dataset") from e
+
+ yield (
+ job_id,
+ True,
+ (
+ total_num_examples,
+ total_num_bytes,
+ writer._features,
+ num_shards,
+ shard_lengths,
+ num_original_shards,
+ original_shard_lengths,
+ ),
+ )
+
+ def _download_and_prepare(self, dl_manager, verification_mode, **prepare_splits_kwargs):
+ super()._download_and_prepare(
+ dl_manager,
+ verification_mode,
+ **prepare_splits_kwargs,
+ )
+
+ def _get_examples_iterable_for_split(self, split_generator: SplitGenerator) -> ExamplesIterable:
+ return ExamplesIterable(
+ self._generate_examples,
+ split_generator.gen_kwargs,
+ generate_more_kwargs_fn=getattr(self, "_generate_more_gen_kwargs", None),
+ )
+
+
+class ArrowBasedBuilder(DatasetBuilder):
+ """Base class for datasets with data generation based on Arrow loading functions (CSV/JSON/Parquet)."""
+
+ def _generate_shards(self, **kwargs) -> Iterator[Union[str, dict[str, Any]]]:
+ """Default function generating shards paths for each `SplitGenerator`.
+
+ This function is useful to list the original shards from where the data
+ comes from and is either converted to Arrow or streamed to an IterableDataset.
+
+ This is optional and only used for certain utilities, but not in Dataset
+ nor IterableDataset. E.g. it's used to map original shard files to Parquet
+ files in the Dataset Viewer after conversion.
+
+ Args:
+ **kwargs (additional keyword arguments):
+ Arguments forwarded from the SplitGenerator.gen_kwargs
+
+ Yields:
+ shard: generally a string representing the shard path, or a dict
+ representing the shard in case of shards spanning intra or inter-files.
+ """
+ raise NotImplementedError()
+
+ @abc.abstractmethod
+ def _generate_tables(self, **kwargs) -> Iterator[tuple[Key, pa.Table]]:
+ """Default function generating examples for each `SplitGenerator`.
+
+ This function preprocess the examples from the raw data to the preprocessed
+ dataset files.
+ This function is called once for each `SplitGenerator` defined in
+ `_split_generators`. The examples yielded here will be written on
+ disk.
+
+ Args:
+ **kwargs (additional keyword arguments):
+ Arguments forwarded from the SplitGenerator.gen_kwargs
+
+ Yields:
+ key: tuple[int, int] original_shard_id and table_idx within that shard
+ example: `pyarrow.Table`, a feature table
+ ready to be encoded and written to disk.
+ """
+ raise NotImplementedError()
+
+ def _prepare_split(
+ self,
+ split_generator: SplitGenerator,
+ file_format: str = "arrow",
+ num_proc: Optional[int] = None,
+ max_shard_size: Optional[Union[str, int]] = None,
+ ):
+ max_shard_size = convert_file_size_to_int(max_shard_size or config.MAX_SHARD_SIZE)
+
+ try:
+ split_info = self.info.splits[split_generator.name]
+ except Exception:
+ split_info = split_generator.split_info
+
+ SUFFIX = "-JJJJJ-SSSSS-of-NNNNN"
+ fname = f"{self.dataset_name}-{split_generator.name}{SUFFIX}.{file_format}"
+ fpath = posixpath.join(self._output_dir, fname)
+
+ if num_proc and num_proc > 1:
+ num_original_shards = _number_of_shards_in_gen_kwargs(split_generator.gen_kwargs)
+ if num_original_shards <= 1:
+ logger.warning(
+ f"Setting num_proc from {num_proc} back to 1 for the {split_info.name} split to disable multiprocessing as it only contains one shard."
+ )
+ num_proc = 1
+ elif num_original_shards < num_proc:
+ logger.warning(
+ f"Setting num_proc from {num_proc} to {num_original_shards} for the {split_info.name} split as it only contains {num_original_shards} shards."
+ )
+ num_proc = num_original_shards
+
+ pbar = hf_tqdm(
+ unit=" examples",
+ total=split_info.num_examples,
+ desc=f"Generating {split_info.name} split",
+ )
+
+ _prepare_split_args = {
+ "fpath": fpath,
+ "file_format": file_format,
+ "max_shard_size": max_shard_size,
+ }
+
+ if num_proc is None or num_proc == 1:
+ result = None
+ gen_kwargs = split_generator.gen_kwargs
+ job_id = 0
+ with pbar:
+ for job_id, done, content in self._prepare_split_single(
+ gen_kwargs=gen_kwargs, job_id=job_id, **_prepare_split_args
+ ):
+ if done:
+ result = content
+ else:
+ pbar.update(content)
+ # wrapping everything into lists for consistency with the multiprocessed code path
+ assert result is not None, "Failed to retrieve results from prepare_split"
+ (
+ examples_per_job,
+ bytes_per_job,
+ features_per_job,
+ shards_per_job,
+ shard_lengths_per_job,
+ original_shards_per_job,
+ original_shard_lengths_per_job,
+ ) = ([item] for item in result)
+ else:
+ kwargs_per_job = [
+ {"gen_kwargs": gen_kwargs, "job_id": job_id, **_prepare_split_args}
+ for job_id, gen_kwargs in enumerate(
+ _split_gen_kwargs(split_generator.gen_kwargs, max_num_jobs=num_proc)
+ )
+ ]
+ num_jobs = len(kwargs_per_job)
+
+ examples_per_job = [None] * num_jobs
+ bytes_per_job = [None] * num_jobs
+ features_per_job = [None] * num_jobs
+ shards_per_job = [None] * num_jobs
+ shard_lengths_per_job = [None] * num_jobs
+ original_shards_per_job = [None] * num_jobs
+ original_shard_lengths_per_job = [None] * num_jobs
+
+ with Pool(num_proc) as pool:
+ with pbar:
+ for job_id, done, content in iflatmap_unordered(
+ pool, self._prepare_split_single, kwargs_iterable=kwargs_per_job
+ ):
+ if done:
+ # the content is the result of the job
+ (
+ examples_per_job[job_id],
+ bytes_per_job[job_id],
+ features_per_job[job_id],
+ shards_per_job[job_id],
+ shard_lengths_per_job[job_id],
+ original_shards_per_job[job_id],
+ original_shard_lengths_per_job[job_id],
+ ) = content
+ else:
+ # the content is the number of examples progress update
+ pbar.update(content)
+
+ assert None not in examples_per_job, (
+ f"Failed to retrieve results from prepare_split: result list {examples_per_job} still contains None - at least one worker failed to return its results"
+ )
+
+ total_shards = sum(shards_per_job)
+ total_original_shards = sum(original_shards_per_job)
+ total_num_examples = sum(examples_per_job)
+ total_num_bytes = sum(bytes_per_job)
+ features = features_per_job[0]
+
+ split_generator.split_info.num_examples = total_num_examples
+ split_generator.split_info.num_bytes = total_num_bytes
+
+ # should rename everything at the end
+ logger.debug(f"Renaming {total_shards} shards.")
+ if total_shards > 1:
+ # use the -SSSSS-of-NNNNN pattern
+
+ def _rename_shard(shard_id_and_job: tuple[int]):
+ shard_id, job_id = shard_id_and_job
+ global_shard_id = sum(shards_per_job[:job_id]) + shard_id
+ self._rename(
+ fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
+ fpath.replace("JJJJJ-SSSSS", f"{global_shard_id:05d}").replace("NNNNN", f"{total_shards:05d}"),
+ )
+
+ shard_ids_and_jobs = [
+ (shard_id, job_id)
+ for job_id, num_shards in enumerate(shards_per_job)
+ for shard_id in range(num_shards)
+ ]
+ thread_map(_rename_shard, shard_ids_and_jobs, disable=True, max_workers=64)
+
+ split_generator.split_info.shard_lengths = [
+ shard_length for shard_lengths in shard_lengths_per_job for shard_length in shard_lengths
+ ]
+ else:
+ # don't use any pattern
+ shard_id, job_id = 0, 0
+ self._rename(
+ fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
+ fpath.replace(SUFFIX, ""),
+ )
+
+ if total_original_shards > 1 and config.SAVE_ORIGINAL_SHARD_LENGTHS:
+ split_generator.split_info.original_shard_lengths = [
+ original_shard_length
+ for original_shard_lengths in original_shard_lengths_per_job
+ for original_shard_length in original_shard_lengths
+ ]
+
+ if self.info.features is None:
+ self.info.features = features
+
+ def _prepare_split_single(
+ self, gen_kwargs: dict, fpath: str, file_format: str, max_shard_size: int, job_id: int
+ ) -> Iterator[tuple[int, bool, tuple[int, int, Features, int, int, int]]]:
+ gen_kwargs = {k: tracked_list(v) if isinstance(v, list) else v for k, v in gen_kwargs.items()}
+ generator = self._generate_tables(**gen_kwargs)
+ writer_class = ParquetWriter if file_format == "parquet" else ArrowWriter
+ embed_local_files = file_format == "parquet"
+ shard_lengths = []
+ original_shard_lengths = []
+ total_num_examples, total_num_bytes = 0, 0
+
+ shard_id = 0
+ original_shard_id = 0
+ num_examples_progress_update = 0
+ try:
+ writer = writer_class(
+ features=self.info.features,
+ path=fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
+ writer_batch_size=self._writer_batch_size,
+ storage_options=self._fs.storage_options,
+ embed_local_files=embed_local_files,
+ )
+ try:
+ _time = time.time()
+ for key, table in generator:
+ if isinstance(key, Key): # old custom builders may not use Key
+ original_shard_id = key.original_shard_id
+ if max_shard_size is not None and writer._num_bytes > max_shard_size:
+ num_examples, num_bytes = writer.finalize()
+ writer.close()
+ shard_lengths.append(num_examples)
+ total_num_examples += num_examples
+ total_num_bytes += num_bytes
+ shard_id += 1
+ writer = writer_class(
+ features=writer._features,
+ path=fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
+ writer_batch_size=self._writer_batch_size,
+ storage_options=self._fs.storage_options,
+ embed_local_files=embed_local_files,
+ )
+ try:
+ if len(table) == 1:
+ writer.write_row(table)
+ else:
+ writer.write_table(table)
+ except CastError as cast_error:
+ raise DatasetGenerationCastError.from_cast_error(
+ cast_error=cast_error,
+ builder_name=self.info.builder_name,
+ gen_kwargs=gen_kwargs,
+ token=self.token,
+ )
+ if len(original_shard_lengths) == original_shard_id:
+ original_shard_lengths.append(len(table))
+ else:
+ original_shard_lengths[original_shard_id] += len(table)
+ num_examples_progress_update += len(table)
+ if time.time() > _time + config.PBAR_REFRESH_TIME_INTERVAL:
+ _time = time.time()
+ yield job_id, False, num_examples_progress_update
+ num_examples_progress_update = 0
+ finally:
+ yield job_id, False, num_examples_progress_update
+ num_shards = shard_id + 1
+ num_original_shards = original_shard_id + 1
+ num_examples, num_bytes = writer.finalize()
+ writer.close()
+ shard_lengths.append(num_examples)
+ total_num_examples += num_examples
+ total_num_bytes += num_bytes
+ except Exception as e:
+ # Ignore the writer's error for no examples written to the file if this error was caused by the error in _generate_examples before the first example was yielded
+ if isinstance(e, SchemaInferenceError) and e.__context__ is not None:
+ e = e.__context__
+ if isinstance(e, DatasetGenerationError):
+ raise
+ raise DatasetGenerationError("An error occurred while generating the dataset") from e
+
+ yield (
+ job_id,
+ True,
+ (
+ total_num_examples,
+ total_num_bytes,
+ writer._features,
+ num_shards,
+ shard_lengths,
+ num_original_shards,
+ original_shard_lengths,
+ ),
+ )
+
+ def _get_examples_iterable_for_split(self, split_generator: SplitGenerator) -> ExamplesIterable:
+ return ArrowExamplesIterable(
+ self._generate_tables,
+ kwargs=split_generator.gen_kwargs,
+ generate_more_kwargs_fn=getattr(self, "_generate_more_gen_kwargs", None),
+ sleep_on_threads_shutdown=self.SLEEP_ON_THREADS_SHUTDOWNS,
+ )
+
+
+class _CountableBuilderMixin(DatasetBuilder):
+ @abc.abstractmethod
+ def _generate_num_examples(self, **kwargs) -> Iterator[int]:
+ raise NotImplementedError()
+
+ def count_examples(self, dl_manager: DownloadManager) -> dict[str, int]:
+ split_generators_kwargs = self._make_split_generators_kwargs({})
+ split_generators: list[SplitGenerator] = self._split_generators(dl_manager, **split_generators_kwargs)
+ return {split_generator.name: self._count_examples(split_generator) for split_generator in split_generators}
+
+ def _count_examples(self, split_generator: SplitGenerator) -> int:
+ max_workers = min(32, os.cpu_count() + 4)
+ return sum(
+ thread_map(
+ self._count_examples_single,
+ _split_gen_kwargs(split_generator.gen_kwargs, max_workers),
+ delay=5,
+ desc=f"Counting rows for split={split_generator.name}",
+ )
+ )
+
+ def _count_examples_single(self, gen_kwargs: dict[str, Any]) -> int:
+ return sum(self._generate_num_examples(**gen_kwargs))
diff --git a/venv/Lib/site-packages/datasets/combine.py b/venv/Lib/site-packages/datasets/combine.py
new file mode 100644
index 0000000000000000000000000000000000000000..7b6c64b4cc16d83ac57bb9c4f6883e7ada7f19f7
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/combine.py
@@ -0,0 +1,232 @@
+from typing import Optional, TypeVar
+
+from .arrow_dataset import Dataset, _concatenate_map_style_datasets, _interleave_map_style_datasets
+from .dataset_dict import DatasetDict, IterableDatasetDict
+from .info import DatasetInfo
+from .iterable_dataset import IterableDataset, _concatenate_iterable_datasets, _interleave_iterable_datasets
+from .splits import NamedSplit
+from .utils import logging
+from .utils.py_utils import Literal
+
+
+logger = logging.get_logger(__name__)
+
+
+DatasetType = TypeVar("DatasetType", Dataset, IterableDataset)
+
+
+def interleave_datasets(
+ datasets: list[DatasetType],
+ probabilities: Optional[list[float]] = None,
+ seed: Optional[int] = None,
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ stopping_strategy: Literal[
+ "first_exhausted", "all_exhausted", "all_exhausted_without_replacement"
+ ] = "first_exhausted",
+) -> DatasetType:
+ """
+ Interleave several datasets (sources) into a single dataset.
+ The new dataset is constructed by alternating between the sources to get the examples.
+
+ You can use this function on a list of [`Dataset`] objects, or on a list of [`IterableDataset`] objects.
+
+ - If `probabilities` is `None` (default) the new dataset is constructed by cycling between each source to get the examples.
+ - If `probabilities` is not `None`, the new dataset is constructed by getting examples from a random source at a time according to the provided probabilities.
+
+ The resulting dataset ends when one of the source datasets runs out of examples except when `oversampling` is `True`,
+ in which case, the resulting dataset ends when all datasets have ran out of examples at least one time.
+
+ Note for iterable datasets:
+
+ * The resulting dataset's `num_shards` is the minimum of each dataset's `num_shards` to ensure good parallelism.
+ If some of your datasets have a very low number of shards, you may use [`IterableDataset.reshard`].
+ * In a distributed setup or in PyTorch DataLoader workers, the stopping strategy is applied per process.
+ Therefore the "first_exhausted" strategy on an sharded iterable dataset can generate less samples in total (up to 1 missing sample per subdataset per worker).
+
+ Args:
+ datasets (`List[Dataset]` or `List[IterableDataset]`):
+ List of datasets to interleave.
+ probabilities (`List[float]`, *optional*, defaults to `None`):
+ If specified, the new dataset is constructed by sampling
+ examples from one source at a time according to these probabilities.
+ seed (`int`, *optional*, defaults to `None`):
+ The random seed used to choose a source for each example.
+ info ([`DatasetInfo`], *optional*):
+ Dataset information, like description, citation, etc.
+
+ split ([`NamedSplit`], *optional*):
+ Name of the dataset split.
+
+ stopping_strategy (`str`, defaults to `first_exhausted`):
+ Three strategies are proposed right now, `first_exhausted`, `all_exhausted` and `all_exhausted_without_replacement`.
+ By default, `first_exhausted` is an undersampling strategy, i.e the dataset construction is stopped as soon as one dataset has ran out of samples.
+ If the strategy is `all_exhausted`, we use an oversampling strategy, i.e the dataset construction is stopped as soon as every samples of every dataset has been added at least once.
+ When strategy is `all_exhausted_without_replacement` we make sure that each sample in each dataset is sampled only once.
+ Note that if the strategy is `all_exhausted`, the interleaved dataset size can get enormous:
+ - with no probabilities, the resulting dataset will have `max_length_datasets*nb_dataset` samples.
+ - with given probabilities, the resulting dataset will have more samples if some datasets have really low probability of visiting.
+ Returns:
+ [`Dataset`] or [`IterableDataset`]: Return type depends on the input `datasets`
+ parameter. `Dataset` if the input is a list of `Dataset`, `IterableDataset` if the input is a list of
+ `IterableDataset`.
+
+ Example:
+
+ For regular datasets (map-style):
+
+ ```python
+ >>> from datasets import Dataset, interleave_datasets
+ >>> d1 = Dataset.from_dict({"a": [0, 1, 2]})
+ >>> d2 = Dataset.from_dict({"a": [10, 11, 12]})
+ >>> d3 = Dataset.from_dict({"a": [20, 21, 22]})
+ >>> dataset = interleave_datasets([d1, d2, d3], probabilities=[0.7, 0.2, 0.1], seed=42, stopping_strategy="all_exhausted")
+ >>> dataset["a"]
+ [10, 0, 11, 1, 2, 20, 12, 10, 0, 1, 2, 21, 0, 11, 1, 2, 0, 1, 12, 2, 10, 0, 22]
+ >>> dataset = interleave_datasets([d1, d2, d3], probabilities=[0.7, 0.2, 0.1], seed=42)
+ >>> dataset["a"]
+ [10, 0, 11, 1, 2]
+ >>> dataset = interleave_datasets([d1, d2, d3])
+ >>> dataset["a"]
+ [0, 10, 20, 1, 11, 21, 2, 12, 22]
+ >>> dataset = interleave_datasets([d1, d2, d3], stopping_strategy="all_exhausted")
+ >>> dataset["a"]
+ [0, 10, 20, 1, 11, 21, 2, 12, 22]
+ >>> d1 = Dataset.from_dict({"a": [0, 1, 2]})
+ >>> d2 = Dataset.from_dict({"a": [10, 11, 12, 13]})
+ >>> d3 = Dataset.from_dict({"a": [20, 21, 22, 23, 24]})
+ >>> dataset = interleave_datasets([d1, d2, d3])
+ >>> dataset["a"]
+ [0, 10, 20, 1, 11, 21, 2, 12, 22]
+ >>> dataset = interleave_datasets([d1, d2, d3], stopping_strategy="all_exhausted")
+ >>> dataset["a"]
+ [0, 10, 20, 1, 11, 21, 2, 12, 22, 0, 13, 23, 1, 10, 24]
+ >>> dataset = interleave_datasets([d1, d2, d3], probabilities=[0.7, 0.2, 0.1], seed=42)
+ >>> dataset["a"]
+ [10, 0, 11, 1, 2]
+ >>> dataset = interleave_datasets([d1, d2, d3], probabilities=[0.7, 0.2, 0.1], seed=42, stopping_strategy="all_exhausted")
+ >>> dataset["a"]
+ [10, 0, 11, 1, 2, 20, 12, 13, ..., 0, 1, 2, 0, 24]
+ For datasets in streaming mode (iterable):
+
+ >>> from datasets import interleave_datasets
+ >>> d1 = load_dataset('allenai/c4', 'es', split='train', streaming=True)
+ >>> d2 = load_dataset('allenai/c4', 'fr', split='train', streaming=True)
+ >>> dataset = interleave_datasets([d1, d2])
+ >>> iterator = iter(dataset)
+ >>> next(iterator)
+ {'text': 'Comprar Zapatillas para niña en chancla con goma por...'}
+ >>> next(iterator)
+ {'text': 'Le sacre de philippe ier, 23 mai 1059 - Compte Rendu...'
+ ```
+ """
+ from .arrow_dataset import Dataset
+ from .iterable_dataset import IterableDataset
+
+ if not datasets:
+ raise ValueError("Unable to interleave an empty list of datasets.")
+ for i, dataset in enumerate(datasets):
+ if not isinstance(dataset, (Dataset, IterableDataset)):
+ if isinstance(dataset, (DatasetDict, IterableDatasetDict)):
+ if not dataset:
+ raise ValueError(
+ f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} "
+ "is an empty dataset dictionary."
+ )
+ raise ValueError(
+ f"Dataset at position {i} has at least one split: {list(dataset)}\n"
+ f"Please pick one to interleave with the other datasets, for example: dataset['{next(iter(dataset))}']"
+ )
+ raise ValueError(
+ f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(dataset).__name__}."
+ )
+ if i == 0:
+ dataset_type, other_type = (
+ (Dataset, IterableDataset) if isinstance(dataset, Dataset) else (IterableDataset, Dataset)
+ )
+ elif not isinstance(dataset, dataset_type):
+ raise ValueError(
+ f"Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects."
+ )
+ if stopping_strategy not in ["first_exhausted", "all_exhausted", "all_exhausted_without_replacement"]:
+ raise ValueError(f"{stopping_strategy} is not supported. Please enter a valid stopping_strategy.")
+ if dataset_type is Dataset:
+ return _interleave_map_style_datasets(
+ datasets, probabilities, seed, info=info, split=split, stopping_strategy=stopping_strategy
+ )
+ else:
+ return _interleave_iterable_datasets(
+ datasets,
+ probabilities,
+ seed,
+ info=info,
+ split=split,
+ stopping_strategy=stopping_strategy,
+ )
+
+
+def concatenate_datasets(
+ dsets: list[DatasetType],
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ axis: int = 0,
+) -> DatasetType:
+ """
+ Concatenate several datasets (sources) into a single dataset.
+
+ Use axis=0 to concatenate vertically (default), or axis=1 to concatenate horizontally.
+
+ Note for iterable datasets:
+
+ * if axis=0, the resulting dataset's `num_shards` is the sum of each dataset's `num_shards`.
+ * if axis=1, the resulting dataset has one (1) shard to not misalign data.
+
+ Args:
+ dsets (`List[datasets.Dataset]` or `List[datasets.IterableDataset]`):
+ List of Datasets to concatenate.
+ info (`DatasetInfo`, *optional*):
+ Dataset information, like description, citation, etc.
+ split (`NamedSplit`, *optional*):
+ Name of the dataset split.
+ axis (`{0, 1}`, defaults to `0`):
+ Axis to concatenate over, where `0` means over rows (vertically) and `1` means over columns
+ (horizontally).
+
+
+
+ Example:
+
+ ```py
+ >>> ds3 = concatenate_datasets([ds1, ds2])
+ ```
+ """
+
+ if not dsets:
+ raise ValueError("Unable to concatenate an empty list of datasets.")
+ for i, dataset in enumerate(dsets):
+ if not isinstance(dataset, (Dataset, IterableDataset)):
+ if isinstance(dataset, (DatasetDict, IterableDatasetDict)):
+ if not dataset:
+ raise ValueError(
+ f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} "
+ "is an empty dataset dictionary."
+ )
+ raise ValueError(
+ f"Dataset at position {i} has at least one split: {list(dataset)}\n"
+ f"Please pick one to interleave with the other datasets, for example: dataset['{next(iter(dataset))}']"
+ )
+ raise ValueError(
+ f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(dataset).__name__}."
+ )
+ if i == 0:
+ dataset_type, other_type = (
+ (Dataset, IterableDataset) if isinstance(dataset, Dataset) else (IterableDataset, Dataset)
+ )
+ elif not isinstance(dataset, dataset_type):
+ raise ValueError(
+ f"Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects."
+ )
+ if dataset_type is Dataset:
+ return _concatenate_map_style_datasets(dsets, info=info, split=split, axis=axis)
+ else:
+ return _concatenate_iterable_datasets(dsets, info=info, split=split, axis=axis)
diff --git a/venv/Lib/site-packages/datasets/commands/__init__.py b/venv/Lib/site-packages/datasets/commands/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..905e753955a348a8e486302e1b6f5e8f53ec7bf4
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/commands/__init__.py
@@ -0,0 +1,13 @@
+from abc import ABC, abstractmethod
+from argparse import ArgumentParser
+
+
+class BaseDatasetsCLICommand(ABC):
+ @staticmethod
+ @abstractmethod
+ def register_subcommand(parser: ArgumentParser):
+ raise NotImplementedError()
+
+ @abstractmethod
+ def run(self):
+ raise NotImplementedError()
diff --git a/venv/Lib/site-packages/datasets/commands/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/commands/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..2969d45146b13354b233cd568aeb1055b6b8abc4
Binary files /dev/null and b/venv/Lib/site-packages/datasets/commands/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/commands/__pycache__/datasets_cli.cpython-311.pyc b/venv/Lib/site-packages/datasets/commands/__pycache__/datasets_cli.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..30ae00bc8d2e319a05f6f0515e17ee4220ce0a18
Binary files /dev/null and b/venv/Lib/site-packages/datasets/commands/__pycache__/datasets_cli.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/commands/__pycache__/delete_from_hub.cpython-311.pyc b/venv/Lib/site-packages/datasets/commands/__pycache__/delete_from_hub.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..77788c3d7d990f1252db25949d667f52dda0375d
Binary files /dev/null and b/venv/Lib/site-packages/datasets/commands/__pycache__/delete_from_hub.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/commands/__pycache__/env.cpython-311.pyc b/venv/Lib/site-packages/datasets/commands/__pycache__/env.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..5b955bec50624b4e95e7ee18c3c545a98f5052d5
Binary files /dev/null and b/venv/Lib/site-packages/datasets/commands/__pycache__/env.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/commands/__pycache__/test.cpython-311.pyc b/venv/Lib/site-packages/datasets/commands/__pycache__/test.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..34c7dee8434c5b51eea91705eb42c73f0848ca63
Binary files /dev/null and b/venv/Lib/site-packages/datasets/commands/__pycache__/test.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/commands/datasets_cli.py b/venv/Lib/site-packages/datasets/commands/datasets_cli.py
new file mode 100644
index 0000000000000000000000000000000000000000..1d04d87ede1bfddc0886c0ae1686fa02ebd21768
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/commands/datasets_cli.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+from argparse import ArgumentParser
+
+from datasets.commands.delete_from_hub import DeleteFromHubCommand
+from datasets.commands.env import EnvironmentCommand
+from datasets.commands.test import TestCommand
+from datasets.utils.logging import set_verbosity_info
+
+
+def parse_unknown_args(unknown_args):
+ return {key.lstrip("-"): value for key, value in zip(unknown_args[::2], unknown_args[1::2])}
+
+
+def main():
+ parser = ArgumentParser(
+ "HuggingFace Datasets CLI tool", usage="datasets-cli []", allow_abbrev=False
+ )
+ commands_parser = parser.add_subparsers(help="datasets-cli command helpers")
+ set_verbosity_info()
+
+ # Register commands
+ EnvironmentCommand.register_subcommand(commands_parser)
+ TestCommand.register_subcommand(commands_parser)
+ DeleteFromHubCommand.register_subcommand(commands_parser)
+
+ # Parse args
+ args, unknown_args = parser.parse_known_args()
+ if not hasattr(args, "func"):
+ parser.print_help()
+ exit(1)
+ kwargs = parse_unknown_args(unknown_args)
+
+ # Run
+ service = args.func(args, **kwargs)
+ service.run()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/venv/Lib/site-packages/datasets/commands/delete_from_hub.py b/venv/Lib/site-packages/datasets/commands/delete_from_hub.py
new file mode 100644
index 0000000000000000000000000000000000000000..126a029c246ab584df47c4ba1b908a798b9003b6
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/commands/delete_from_hub.py
@@ -0,0 +1,42 @@
+from argparse import ArgumentParser
+from typing import Optional
+
+from datasets.commands import BaseDatasetsCLICommand
+from datasets.hub import delete_from_hub
+
+
+def _command_factory(args):
+ return DeleteFromHubCommand(
+ args.dataset_id,
+ args.config_name,
+ args.token,
+ args.revision,
+ )
+
+
+class DeleteFromHubCommand(BaseDatasetsCLICommand):
+ @staticmethod
+ def register_subcommand(parser):
+ parser: ArgumentParser = parser.add_parser("delete_from_hub", help="Delete dataset config from the Hub")
+ parser.add_argument(
+ "dataset_id", help="source dataset ID, e.g. USERNAME/DATASET_NAME or ORGANIZATION/DATASET_NAME"
+ )
+ parser.add_argument("config_name", help="config name to delete")
+ parser.add_argument("--token", help="access token to the Hugging Face Hub")
+ parser.add_argument("--revision", help="source revision")
+ parser.set_defaults(func=_command_factory)
+
+ def __init__(
+ self,
+ dataset_id: str,
+ config_name: str,
+ token: Optional[str],
+ revision: Optional[str],
+ ):
+ self._dataset_id = dataset_id
+ self._config_name = config_name
+ self._token = token
+ self._revision = revision
+
+ def run(self) -> None:
+ _ = delete_from_hub(self._dataset_id, self._config_name, revision=self._revision, token=self._token)
diff --git a/venv/Lib/site-packages/datasets/commands/env.py b/venv/Lib/site-packages/datasets/commands/env.py
new file mode 100644
index 0000000000000000000000000000000000000000..40b2a3654c8f1c6c080222a80b12d108972a5dc9
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/commands/env.py
@@ -0,0 +1,41 @@
+import platform
+from argparse import ArgumentParser
+
+import fsspec
+import huggingface_hub
+import pandas
+import pyarrow
+
+from datasets import __version__ as version
+from datasets.commands import BaseDatasetsCLICommand
+
+
+def info_command_factory(_):
+ return EnvironmentCommand()
+
+
+class EnvironmentCommand(BaseDatasetsCLICommand):
+ @staticmethod
+ def register_subcommand(parser: ArgumentParser):
+ download_parser = parser.add_parser("env", help="Print relevant system environment info.")
+ download_parser.set_defaults(func=info_command_factory)
+
+ def run(self):
+ info = {
+ "`datasets` version": version,
+ "Platform": platform.platform(),
+ "Python version": platform.python_version(),
+ "`huggingface_hub` version": huggingface_hub.__version__,
+ "PyArrow version": pyarrow.__version__,
+ "Pandas version": pandas.__version__,
+ "`fsspec` version": fsspec.__version__,
+ }
+
+ print("\nCopy-and-paste the text below in your GitHub issue.\n")
+ print(self.format_dict(info))
+
+ return info
+
+ @staticmethod
+ def format_dict(d):
+ return "\n".join([f"- {prop}: {val}" for prop, val in d.items()]) + "\n"
diff --git a/venv/Lib/site-packages/datasets/commands/test.py b/venv/Lib/site-packages/datasets/commands/test.py
new file mode 100644
index 0000000000000000000000000000000000000000..1aca3253df984fcbee2dec5e82f43e5bb71e6e56
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/commands/test.py
@@ -0,0 +1,180 @@
+import logging
+import os
+from argparse import ArgumentParser
+from collections.abc import Generator
+from shutil import rmtree
+
+import datasets.config
+from datasets.builder import DatasetBuilder
+from datasets.commands import BaseDatasetsCLICommand
+from datasets.download.download_manager import DownloadMode
+from datasets.info import DatasetInfosDict
+from datasets.load import dataset_module_factory, get_dataset_builder_class
+from datasets.utils.info_utils import VerificationMode
+from datasets.utils.logging import ERROR, get_logger
+
+
+logger = get_logger(__name__)
+
+
+def _test_command_factory(args):
+ return TestCommand(
+ args.dataset,
+ args.name,
+ args.cache_dir,
+ args.data_dir,
+ args.all_configs,
+ args.save_info or args.save_infos,
+ args.ignore_verifications,
+ args.force_redownload,
+ args.clear_cache,
+ args.num_proc,
+ )
+
+
+class TestCommand(BaseDatasetsCLICommand):
+ __test__ = False # to tell pytest it's not a test class
+
+ @staticmethod
+ def register_subcommand(parser: ArgumentParser):
+ test_parser = parser.add_parser("test", help="Test dataset loading.")
+ test_parser.add_argument("--name", type=str, default=None, help="Dataset processing name")
+ test_parser.add_argument(
+ "--cache_dir",
+ type=str,
+ default=None,
+ help="Cache directory where the datasets are stored.",
+ )
+ test_parser.add_argument(
+ "--data_dir",
+ type=str,
+ default=None,
+ help="Can be used to specify a manual directory to get the files from.",
+ )
+ test_parser.add_argument("--all_configs", action="store_true", help="Test all dataset configurations")
+ test_parser.add_argument(
+ "--save_info", action="store_true", help="Save the dataset infos in the dataset card (README.md)"
+ )
+ test_parser.add_argument(
+ "--ignore_verifications",
+ action="store_true",
+ help="Run the test without checksums and splits checks.",
+ )
+ test_parser.add_argument("--force_redownload", action="store_true", help="Force dataset redownload")
+ test_parser.add_argument(
+ "--clear_cache",
+ action="store_true",
+ help="Remove downloaded files and cached datasets after each config test",
+ )
+ test_parser.add_argument("--num_proc", type=int, default=None, help="Number of processes")
+ # aliases
+ test_parser.add_argument("--save_infos", action="store_true", help="alias to save_info")
+ test_parser.add_argument("dataset", type=str, help="Name of the dataset to download")
+ test_parser.set_defaults(func=_test_command_factory)
+
+ def __init__(
+ self,
+ dataset: str,
+ name: str,
+ cache_dir: str,
+ data_dir: str,
+ all_configs: bool,
+ save_infos: bool,
+ ignore_verifications: bool,
+ force_redownload: bool,
+ clear_cache: bool,
+ num_proc: int,
+ ):
+ self._dataset = dataset
+ self._name = name
+ self._cache_dir = cache_dir
+ self._data_dir = data_dir
+ self._all_configs = all_configs
+ self._save_infos = save_infos
+ self._ignore_verifications = ignore_verifications
+ self._force_redownload = force_redownload
+ self._clear_cache = clear_cache
+ self._num_proc = num_proc
+ if clear_cache and not cache_dir:
+ print(
+ "When --clear_cache is used, specifying a cache directory is mandatory.\n"
+ "The 'download' folder of the cache directory and the dataset builder cache will be deleted after each configuration test.\n"
+ "Please provide a --cache_dir that will be used to test the dataset."
+ )
+ exit(1)
+ if save_infos:
+ self._ignore_verifications = True
+
+ def run(self):
+ logging.getLogger("filelock").setLevel(ERROR)
+ if self._name is not None and self._all_configs:
+ print("Both parameters `config` and `all_configs` can't be used at once.")
+ exit(1)
+ path, config_name = self._dataset, self._name
+ module = dataset_module_factory(path)
+ builder_cls = get_dataset_builder_class(module)
+ n_builders = len(builder_cls.BUILDER_CONFIGS) if self._all_configs and builder_cls.BUILDER_CONFIGS else 1
+
+ def get_builders() -> Generator[DatasetBuilder, None, None]:
+ if self._all_configs and builder_cls.BUILDER_CONFIGS:
+ for i, config in enumerate(builder_cls.BUILDER_CONFIGS):
+ if "config_name" in module.builder_kwargs:
+ yield builder_cls(
+ cache_dir=self._cache_dir,
+ data_dir=self._data_dir,
+ **module.builder_kwargs,
+ )
+ else:
+ yield builder_cls(
+ config_name=config.name,
+ cache_dir=self._cache_dir,
+ data_dir=self._data_dir,
+ **module.builder_kwargs,
+ )
+ else:
+ if "config_name" in module.builder_kwargs:
+ yield builder_cls(cache_dir=self._cache_dir, data_dir=self._data_dir, **module.builder_kwargs)
+ else:
+ yield builder_cls(
+ config_name=config_name,
+ cache_dir=self._cache_dir,
+ data_dir=self._data_dir,
+ **module.builder_kwargs,
+ )
+
+ for j, builder in enumerate(get_builders()):
+ print(f"Testing builder '{builder.config.name}' ({j + 1}/{n_builders})")
+ builder._record_checksums = os.path.exists(
+ os.path.join(builder.get_imported_module_dir(), datasets.config.DATASETDICT_INFOS_FILENAME)
+ ) # record checksums only if we need to update a (deprecated) dataset_infos.json
+ builder.download_and_prepare(
+ download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS
+ if not self._force_redownload
+ else DownloadMode.FORCE_REDOWNLOAD,
+ verification_mode=VerificationMode.NO_CHECKS
+ if self._ignore_verifications
+ else VerificationMode.ALL_CHECKS,
+ num_proc=self._num_proc,
+ )
+ builder.as_dataset()
+
+ # If save_infos=True, we create the dataset card (README.md)
+ # The dataset_infos are saved in the YAML part of the README.md
+ # This is to allow the user to upload them on HF afterwards.
+ if self._save_infos:
+ save_infos_dir = os.path.basename(path) if not os.path.isdir(path) else path
+ os.makedirs(save_infos_dir, exist_ok=True)
+ DatasetInfosDict(**{builder.config.name: builder.info}).write_to_directory(save_infos_dir)
+ print(f"Dataset card saved at {os.path.join(save_infos_dir, datasets.config.REPOCARD_FILENAME)}")
+
+ # If clear_cache=True, the download folder and the dataset builder cache directory are deleted
+ if self._clear_cache:
+ if os.path.isdir(builder._cache_dir):
+ logger.warning(f"Clearing cache at {builder._cache_dir}")
+ rmtree(builder._cache_dir)
+ download_dir = os.path.join(self._cache_dir, datasets.config.DOWNLOADED_DATASETS_DIR)
+ if os.path.isdir(download_dir):
+ logger.warning(f"Clearing cache at {download_dir}")
+ rmtree(download_dir)
+
+ print("Test successful.")
diff --git a/venv/Lib/site-packages/datasets/config.py b/venv/Lib/site-packages/datasets/config.py
new file mode 100644
index 0000000000000000000000000000000000000000..5bbd646caea08e3885df06f090f0ef7ecdd9138e
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/config.py
@@ -0,0 +1,277 @@
+import importlib
+import importlib.metadata
+import logging
+import os
+import platform
+from pathlib import Path
+from typing import Optional
+
+from huggingface_hub import constants
+from packaging import version
+
+
+logger = logging.getLogger(__name__.split(".", 1)[0]) # to avoid circular import from .utils.logging
+
+# Datasets
+S3_DATASETS_BUCKET_PREFIX = "https://s3.amazonaws.com/datasets.huggingface.co/datasets/datasets"
+CLOUDFRONT_DATASETS_DISTRIB_PREFIX = "https://cdn-datasets.huggingface.co/datasets/datasets"
+REPO_DATASETS_URL = "https://raw.githubusercontent.com/huggingface/datasets/{revision}/datasets/{path}/{name}"
+
+# Hub
+HF_ENDPOINT = os.environ.get("HF_ENDPOINT", "https://huggingface.co")
+HUB_DATASETS_URL = HF_ENDPOINT + "/datasets/{repo_id}/resolve/{revision}/{path}"
+HUB_DATASETS_HFFS_URL = "hf://datasets/{repo_id}@{revision}/{path}"
+HUB_DEFAULT_VERSION = "main"
+
+PY_VERSION = version.parse(platform.python_version())
+
+# General environment variables accepted values for booleans
+ENV_VARS_TRUE_VALUES = {"1", "ON", "YES", "TRUE"}
+ENV_VARS_FALSE_VALUES = {"0", "OFF", "NO", "FALSE"}
+ENV_VARS_TRUE_AND_AUTO_VALUES = ENV_VARS_TRUE_VALUES.union({"AUTO"})
+ENV_VARS_FALSE_AND_AUTO_VALUES = ENV_VARS_FALSE_VALUES.union({"AUTO"})
+
+
+# Imports
+DILL_VERSION = version.parse(importlib.metadata.version("dill"))
+FSSPEC_VERSION = version.parse(importlib.metadata.version("fsspec"))
+PANDAS_VERSION = version.parse(importlib.metadata.version("pandas"))
+PYARROW_VERSION = version.parse(importlib.metadata.version("pyarrow"))
+HF_HUB_VERSION = version.parse(importlib.metadata.version("huggingface_hub"))
+
+USE_TF = os.environ.get("USE_TF", "AUTO").upper()
+USE_TORCH = os.environ.get("USE_TORCH", "AUTO").upper()
+USE_JAX = os.environ.get("USE_JAX", "AUTO").upper()
+
+TORCH_VERSION = "N/A"
+TORCH_AVAILABLE = False
+
+if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES:
+ TORCH_AVAILABLE = importlib.util.find_spec("torch") is not None
+ if TORCH_AVAILABLE:
+ try:
+ TORCH_VERSION = version.parse(importlib.metadata.version("torch"))
+ logger.debug(f"PyTorch version {TORCH_VERSION} available.")
+ except importlib.metadata.PackageNotFoundError:
+ pass
+else:
+ logger.info("Disabling PyTorch because USE_TF is set")
+
+POLARS_VERSION = "N/A"
+POLARS_AVAILABLE = importlib.util.find_spec("polars") is not None
+
+if POLARS_AVAILABLE:
+ try:
+ POLARS_VERSION = version.parse(importlib.metadata.version("polars"))
+ logger.debug(f"Polars version {POLARS_VERSION} available.")
+ except importlib.metadata.PackageNotFoundError:
+ pass
+
+
+DUCKDB_VERSION = "N/A"
+DUCKDB_AVAILABLE = importlib.util.find_spec("duckdb") is not None
+
+if DUCKDB_AVAILABLE:
+ try:
+ DUCKDB_VERSION = version.parse(importlib.metadata.version("duckdb"))
+ logger.debug(f"Duckdb version {DUCKDB_VERSION} available.")
+ except importlib.metadata.PackageNotFoundError:
+ pass
+
+TF_VERSION = "N/A"
+TF_AVAILABLE = False
+
+if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VALUES:
+ TF_AVAILABLE = importlib.util.find_spec("tensorflow") is not None
+ if TF_AVAILABLE:
+ # For the metadata, we have to look for both tensorflow and tensorflow-cpu
+ for package in [
+ "tensorflow",
+ "tensorflow-cpu",
+ "tensorflow-gpu",
+ "tf-nightly",
+ "tf-nightly-cpu",
+ "tf-nightly-gpu",
+ "intel-tensorflow",
+ "tensorflow-rocm",
+ "tensorflow-macos",
+ ]:
+ try:
+ TF_VERSION = version.parse(importlib.metadata.version(package))
+ except importlib.metadata.PackageNotFoundError:
+ continue
+ else:
+ break
+ else:
+ TF_AVAILABLE = False
+ if TF_AVAILABLE:
+ if TF_VERSION.major < 2:
+ logger.info(f"TensorFlow found but with version {TF_VERSION}. `datasets` requires version 2 minimum.")
+ TF_AVAILABLE = False
+ else:
+ logger.info(f"TensorFlow version {TF_VERSION} available.")
+else:
+ logger.info("Disabling Tensorflow because USE_TORCH is set")
+
+
+JAX_VERSION = "N/A"
+JAX_AVAILABLE = False
+
+if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES:
+ JAX_AVAILABLE = importlib.util.find_spec("jax") is not None and importlib.util.find_spec("jaxlib") is not None
+ if JAX_AVAILABLE:
+ try:
+ JAX_VERSION = version.parse(importlib.metadata.version("jax"))
+ logger.info(f"JAX version {JAX_VERSION} available.")
+ except importlib.metadata.PackageNotFoundError:
+ pass
+else:
+ logger.info("Disabling JAX because USE_JAX is set to False")
+
+
+# Optional tools for data loading
+SQLALCHEMY_AVAILABLE = importlib.util.find_spec("sqlalchemy") is not None
+
+# Optional tools for file parsing and feature decoding
+PIL_AVAILABLE = importlib.util.find_spec("PIL") is not None
+IS_OPUS_SUPPORTED = True
+IS_MP3_SUPPORTED = True
+TORCHCODEC_AVAILABLE = importlib.util.find_spec("torchcodec") is not None
+TORCHVISION_AVAILABLE = importlib.util.find_spec("torchvision") is not None
+PDFPLUMBER_AVAILABLE = importlib.util.find_spec("pdfplumber") is not None
+NIBABEL_AVAILABLE = importlib.util.find_spec("nibabel") is not None
+TRIMESH_AVAILABLE = importlib.util.find_spec("trimesh") is not None
+TEICH_AVAILABLE = importlib.util.find_spec("teich") is not None
+
+# Optional compression tools
+RARFILE_AVAILABLE = importlib.util.find_spec("rarfile") is not None
+ZSTANDARD_AVAILABLE = importlib.util.find_spec("zstandard") is not None
+LZ4_AVAILABLE = importlib.util.find_spec("lz4") is not None
+PY7ZR_AVAILABLE = importlib.util.find_spec("py7zr") is not None
+
+# Cache location
+DEFAULT_XDG_CACHE_HOME = "~/.cache"
+XDG_CACHE_HOME = os.getenv("XDG_CACHE_HOME", DEFAULT_XDG_CACHE_HOME)
+DEFAULT_HF_CACHE_HOME = os.path.join(XDG_CACHE_HOME, "huggingface")
+HF_CACHE_HOME = os.path.expanduser(os.getenv("HF_HOME", DEFAULT_HF_CACHE_HOME))
+
+DEFAULT_HF_DATASETS_CACHE = os.path.join(HF_CACHE_HOME, "datasets")
+HF_DATASETS_CACHE = Path(os.getenv("HF_DATASETS_CACHE", DEFAULT_HF_DATASETS_CACHE))
+
+DEFAULT_HF_MODULES_CACHE = os.path.join(HF_CACHE_HOME, "modules")
+HF_MODULES_CACHE = Path(os.getenv("HF_MODULES_CACHE", DEFAULT_HF_MODULES_CACHE))
+
+DOWNLOADED_DATASETS_DIR = "downloads"
+DEFAULT_DOWNLOADED_DATASETS_PATH = os.path.join(HF_DATASETS_CACHE, DOWNLOADED_DATASETS_DIR)
+DOWNLOADED_DATASETS_PATH = Path(os.getenv("HF_DATASETS_DOWNLOADED_DATASETS_PATH", DEFAULT_DOWNLOADED_DATASETS_PATH))
+
+EXTRACTED_DATASETS_DIR = "extracted"
+DEFAULT_EXTRACTED_DATASETS_PATH = os.path.join(DEFAULT_DOWNLOADED_DATASETS_PATH, EXTRACTED_DATASETS_DIR)
+EXTRACTED_DATASETS_PATH = Path(os.getenv("HF_DATASETS_EXTRACTED_DATASETS_PATH", DEFAULT_EXTRACTED_DATASETS_PATH))
+
+# Cached dataset info options
+SAVE_ORIGINAL_SHARD_LENGTHS = False
+
+# Download count for the website
+HF_UPDATE_DOWNLOAD_COUNTS = (
+ os.environ.get("HF_UPDATE_DOWNLOAD_COUNTS", "AUTO").upper() in ENV_VARS_TRUE_AND_AUTO_VALUES
+)
+
+# For downloads and to check remote files metadata
+HF_DATASETS_MULTITHREADING_MAX_WORKERS = 16
+
+# Dataset viewer API
+USE_PARQUET_EXPORT = True
+
+# Batch size constants. For more info, see:
+# https://github.com/apache/arrow/blob/master/docs/source/cpp/arrays.rst#size-limitations-and-recommendations)
+DEFAULT_MAX_BATCH_SIZE = 1000
+
+DEFAULT_CDC_OPTIONS = {"min_chunk_size": 256 * 1024, "max_chunk_size": 1024 * 1024, "norm_level": 0}
+
+# Size of the preloaded record batch in `Dataset.__iter__`
+ARROW_READER_BATCH_SIZE_IN_DATASET_ITER = 10
+
+# Max uncompressed shard size in bytes (e.g. to shard parquet datasets in push_to_hub or download_and_prepare)
+MAX_SHARD_SIZE = "500MB"
+
+# Max uncompressed row group size in bytes (e.g. for parquet files in push_to_hub or download_and_prepare)
+MAX_ROW_GROUP_SIZE = "100MB"
+
+# Parquet configuration
+PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS = None
+PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS = None
+PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS = None
+PARQUET_ROW_GROUP_SIZE_FOR_VIDEO_DATASETS = None
+
+# Arrow configuration
+ARROW_RECORD_BATCH_SIZE_FOR_AUDIO_DATASETS = 100
+ARROW_RECORD_BATCH_SIZE_FOR_IMAGE_DATASETS = 100
+ARROW_RECORD_BATCH_SIZE_FOR_BINARY_DATASETS = 100
+ARROW_RECORD_BATCH_SIZE_FOR_VIDEO_DATASETS = 10
+
+# Offline mode
+_offline = os.environ.get("HF_DATASETS_OFFLINE")
+HF_HUB_OFFLINE = constants.HF_HUB_OFFLINE if _offline is None else _offline.upper() in ENV_VARS_TRUE_VALUES
+HF_DATASETS_OFFLINE = HF_HUB_OFFLINE # kept for backward-compatibility
+
+# Here, `True` will disable progress bars globally without possibility of enabling it
+# programmatically. `False` will enable them without possibility of disabling them.
+# If environment variable is not set (None), then the user is free to enable/disable
+# them programmatically.
+# TL;DR: env variable has priority over code
+__HF_DATASETS_DISABLE_PROGRESS_BARS = os.environ.get("HF_DATASETS_DISABLE_PROGRESS_BARS")
+HF_DATASETS_DISABLE_PROGRESS_BARS: Optional[bool] = (
+ __HF_DATASETS_DISABLE_PROGRESS_BARS.upper() in ENV_VARS_TRUE_VALUES
+ if __HF_DATASETS_DISABLE_PROGRESS_BARS is not None
+ else None
+)
+
+# In-memory
+DEFAULT_IN_MEMORY_MAX_SIZE = 0 # Disabled
+IN_MEMORY_MAX_SIZE = float(os.environ.get("HF_DATASETS_IN_MEMORY_MAX_SIZE", DEFAULT_IN_MEMORY_MAX_SIZE))
+
+# File names
+DATASET_ARROW_FILENAME = "dataset.arrow"
+DATASET_INDICES_FILENAME = "indices.arrow"
+DATASET_STATE_JSON_FILENAME = "state.json"
+DATASET_INFO_FILENAME = "dataset_info.json"
+DATASETDICT_INFOS_FILENAME = "dataset_infos.json"
+LICENSE_FILENAME = "LICENSE"
+DATASETDICT_JSON_FILENAME = "dataset_dict.json"
+METADATA_CONFIGS_FIELD = "configs"
+REPOCARD_FILENAME = "README.md"
+REPOYAML_FILENAME = ".huggingface.yaml"
+
+MODULE_NAME_FOR_DYNAMIC_MODULES = "datasets_modules"
+
+MAX_DATASET_CONFIG_ID_READABLE_LENGTH = 255
+
+# Temporary cache directory prefix
+TEMP_CACHE_DIR_PREFIX = "hf_datasets-"
+
+# Streaming
+STREAMING_READ_MAX_RETRIES = 20
+STREAMING_READ_RETRY_INTERVAL = 5
+STREAMING_READ_SERVER_UNAVAILABLE_RETRY_INTERVAL = 20
+STREAMING_READ_RATE_LIMIT_RETRY_INTERVAL = 60
+STREAMING_OPEN_MAX_RETRIES = 20
+STREAMING_OPEN_RETRY_INTERVAL = 5
+
+# Datasets repositories exploration
+ARCHIVES_MAX_NUMBER_FOR_MODULE_INFERENCE = 10
+
+# Async map functions
+MAX_NUM_RUNNING_ASYNC_MAP_FUNCTIONS_IN_PARALLEL = 1000
+
+# Progress bars
+PBAR_REFRESH_TIME_INTERVAL = 0.05 # 20 progress updates per sec
+
+# Maximum number of uploaded files per commit
+UPLOADS_MAX_NUMBER_PER_COMMIT = 50
+
+# Backward compatibility
+MAX_TABLE_NBYTES_FOR_PICKLING = 4 << 30
+
+# Time to let PyArrow close threads gracefully
+SLEEP_TIME_ON_THREADS_SHUTDOWN = 5
diff --git a/venv/Lib/site-packages/datasets/data_files.py b/venv/Lib/site-packages/datasets/data_files.py
new file mode 100644
index 0000000000000000000000000000000000000000..3e63c3ac65ffcaa6a6be5b9ef55c63a0c74ce4e6
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/data_files.py
@@ -0,0 +1,829 @@
+import os
+import re
+from functools import partial
+from glob import has_magic
+from pathlib import Path, PurePath
+from typing import Callable, Optional, Union
+
+import huggingface_hub
+from fsspec.core import url_to_fs
+from huggingface_hub import HfFileSystem
+from packaging import version
+from tqdm.contrib.concurrent import thread_map
+
+from . import config
+from .download import DownloadConfig
+from .naming import _split_re
+from .splits import Split
+from .utils import logging
+from .utils import tqdm as hf_tqdm
+from .utils.file_utils import _prepare_path_and_storage_options, is_local_path, is_relative_path, xbasename, xjoin
+from .utils.py_utils import string_to_dict
+
+
+SingleOriginMetadata = Union[tuple[str, str], tuple[str], tuple[()]]
+
+
+SANITIZED_DEFAULT_SPLIT = str(Split.TRAIN)
+
+
+logger = logging.get_logger(__name__)
+
+
+class Url(str):
+ pass
+
+
+class EmptyDatasetError(FileNotFoundError):
+ pass
+
+
+SPLIT_PATTERN_SHARDED = "data/{split}-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*"
+
+SPLIT_KEYWORDS = {
+ Split.TRAIN: ["train", "training"],
+ Split.VALIDATION: ["validation", "valid", "dev", "val"],
+ Split.TEST: ["test", "testing", "eval", "evaluation"],
+}
+NON_WORDS_CHARS = "-._ 0-9"
+if config.FSSPEC_VERSION < version.parse("2023.9.0"):
+ KEYWORDS_IN_FILENAME_BASE_PATTERNS = ["**[{sep}/]{keyword}[{sep}]*", "{keyword}[{sep}]*"]
+ KEYWORDS_IN_DIR_NAME_BASE_PATTERNS = [
+ "{keyword}/**",
+ "{keyword}[{sep}]*/**",
+ "**[{sep}/]{keyword}/**",
+ "**[{sep}/]{keyword}[{sep}]*/**",
+ ]
+elif config.FSSPEC_VERSION < version.parse("2023.12.0"):
+ KEYWORDS_IN_FILENAME_BASE_PATTERNS = ["**/*[{sep}/]{keyword}[{sep}]*", "{keyword}[{sep}]*"]
+ KEYWORDS_IN_DIR_NAME_BASE_PATTERNS = [
+ "{keyword}/**/*",
+ "{keyword}[{sep}]*/**/*",
+ "**/*[{sep}/]{keyword}/**/*",
+ "**/*[{sep}/]{keyword}[{sep}]*/**/*",
+ ]
+else:
+ KEYWORDS_IN_FILENAME_BASE_PATTERNS = ["**/{keyword}[{sep}]*", "**/*[{sep}]{keyword}[{sep}]*"]
+ KEYWORDS_IN_DIR_NAME_BASE_PATTERNS = [
+ "**/{keyword}/**",
+ "**/{keyword}[{sep}]*/**",
+ "**/*[{sep}]{keyword}/**",
+ "**/*[{sep}]{keyword}[{sep}]*/**",
+ ]
+
+DEFAULT_SPLITS = [Split.TRAIN, Split.VALIDATION, Split.TEST]
+DEFAULT_PATTERNS_SPLIT_IN_FILENAME = {
+ split: [
+ pattern.format(keyword=keyword, sep=NON_WORDS_CHARS)
+ for keyword in SPLIT_KEYWORDS[split]
+ for pattern in KEYWORDS_IN_FILENAME_BASE_PATTERNS
+ ]
+ for split in DEFAULT_SPLITS
+}
+DEFAULT_PATTERNS_SPLIT_IN_DIR_NAME = {
+ split: [
+ pattern.format(keyword=keyword, sep=NON_WORDS_CHARS)
+ for keyword in SPLIT_KEYWORDS[split]
+ for pattern in KEYWORDS_IN_DIR_NAME_BASE_PATTERNS
+ ]
+ for split in DEFAULT_SPLITS
+}
+
+
+DEFAULT_PATTERNS_ALL = {
+ Split.TRAIN: ["**"],
+}
+
+DEFAULT_PATTERNS_LOGS = {"logs": ["**/*.eval"]}
+
+ALL_SPLIT_PATTERNS = [SPLIT_PATTERN_SHARDED]
+ALL_DEFAULT_PATTERNS = [
+ DEFAULT_PATTERNS_LOGS,
+ DEFAULT_PATTERNS_SPLIT_IN_DIR_NAME,
+ DEFAULT_PATTERNS_SPLIT_IN_FILENAME,
+ DEFAULT_PATTERNS_ALL,
+]
+WILDCARD_CHARACTERS = "*[]"
+FILES_TO_IGNORE = [
+ "README.md",
+ "config.json",
+ "dataset_info.json",
+ "dataset_infos.json",
+ "dummy_data.zip",
+ "dataset_dict.json",
+]
+
+
+def contains_wildcards(pattern: str) -> bool:
+ return any(wildcard_character in pattern for wildcard_character in WILDCARD_CHARACTERS)
+
+
+def sanitize_patterns(patterns: Union[dict, list, str]) -> dict[str, Union[list[str], "DataFilesList"]]:
+ """
+ Take the data_files patterns from the user, and format them into a dictionary.
+ Each key is the name of the split, and each value is a list of data files patterns (paths or urls).
+ The default split is "train".
+
+ Returns:
+ patterns: dictionary of split_name -> list of patterns
+ """
+ if isinstance(patterns, dict):
+ return {str(key): value if isinstance(value, list) else [value] for key, value in patterns.items()}
+ elif isinstance(patterns, str):
+ return {SANITIZED_DEFAULT_SPLIT: [patterns]}
+ elif isinstance(patterns, list):
+ if any(isinstance(pattern, dict) for pattern in patterns):
+ for pattern in patterns:
+ if not (
+ isinstance(pattern, dict)
+ and len(pattern) == 2
+ and "split" in pattern
+ and isinstance(pattern.get("path"), (str, list))
+ ):
+ raise ValueError(
+ "Invalid format for data_files entry. "
+ "Each item must be a dictionary with the structure "
+ "{'split': , 'path': }.\n"
+ f"Received: {pattern}"
+ )
+ splits = [pattern["split"] for pattern in patterns]
+ if len(set(splits)) != len(splits):
+ raise ValueError(f"Some splits are duplicated in data_files: {splits}")
+ return {
+ str(pattern["split"]): pattern["path"] if isinstance(pattern["path"], list) else [pattern["path"]]
+ for pattern in patterns
+ }
+ else:
+ return {SANITIZED_DEFAULT_SPLIT: patterns}
+ else:
+ return sanitize_patterns(list(patterns))
+
+
+def _is_inside_unrequested_special_dir(matched_rel_path: str, pattern: str) -> bool:
+ """
+ When a path matches a pattern, we additionally check if it's inside a special directory
+ we ignore by default (if it starts with a double underscore).
+
+ Users can still explicitly request a filepath inside such a directory if "__pycache__" is
+ mentioned explicitly in the requested pattern.
+
+ Some examples:
+
+ base directory:
+
+ ./
+ └── __pycache__
+ └── b.txt
+
+ >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "**")
+ True
+ >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "*/b.txt")
+ True
+ >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "__pycache__/*")
+ False
+ >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "__*/*")
+ False
+ """
+ # We just need to check if every special directories from the path is present explicitly in the pattern.
+ # Since we assume that the path matches the pattern, it's equivalent to counting that both
+ # the parent path and the parent pattern have the same number of special directories.
+ data_dirs_to_ignore_in_path = [part for part in PurePath(matched_rel_path).parent.parts if part.startswith("__")]
+ data_dirs_to_ignore_in_pattern = [part for part in PurePath(pattern).parent.parts if part.startswith("__")]
+ return len(data_dirs_to_ignore_in_path) != len(data_dirs_to_ignore_in_pattern)
+
+
+def _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(matched_rel_path: str, pattern: str) -> bool:
+ """
+ When a path matches a pattern, we additionally check if it's a hidden file or if it's inside
+ a hidden directory we ignore by default, i.e. if the file name or a parent directory name starts with a dot.
+
+ Users can still explicitly request a filepath that is hidden or is inside a hidden directory
+ if the hidden part is mentioned explicitly in the requested pattern.
+
+ Some examples:
+
+ base directory:
+
+ ./
+ └── .hidden_file.txt
+
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_file.txt", "**")
+ True
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_file.txt", ".*")
+ False
+
+ base directory:
+
+ ./
+ └── .hidden_dir
+ └── a.txt
+
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", "**")
+ True
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", ".*/*")
+ False
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", ".hidden_dir/*")
+ False
+
+ base directory:
+
+ ./
+ └── .hidden_dir
+ └── .hidden_file.txt
+
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", "**")
+ True
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".*/*")
+ True
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".*/.*")
+ False
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".hidden_dir/*")
+ True
+ >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".hidden_dir/.*")
+ False
+ """
+ # We just need to check if every hidden part from the path is present explicitly in the pattern.
+ # Since we assume that the path matches the pattern, it's equivalent to counting that both
+ # the path and the pattern have the same number of hidden parts.
+ hidden_directories_in_path = [
+ part for part in PurePath(matched_rel_path).parts if part.startswith(".") and not set(part) == {"."}
+ ]
+ hidden_directories_in_pattern = [
+ part for part in PurePath(pattern).parts if part.startswith(".") and not set(part) == {"."}
+ ]
+ return len(hidden_directories_in_path) != len(hidden_directories_in_pattern)
+
+
+def _get_data_files_patterns(pattern_resolver: Callable[[str], list[str]]) -> dict[str, list[str]]:
+ """
+ Get the default pattern from a directory or repository by testing all the supported patterns.
+ The first patterns to return a non-empty list of data files is returned.
+
+ In order, it first tests if SPLIT_PATTERN_SHARDED works, otherwise it tests the patterns in ALL_DEFAULT_PATTERNS.
+ """
+ # first check the split patterns like data/{split}-00000-of-00001.parquet
+ for split_pattern in ALL_SPLIT_PATTERNS:
+ pattern = split_pattern.replace("{split}", "*")
+ try:
+ data_files = pattern_resolver(pattern)
+ except FileNotFoundError:
+ continue
+ if len(data_files) > 0:
+ splits: set[str] = set()
+ for p in data_files:
+ p_parts = string_to_dict(xbasename(p), xbasename(split_pattern))
+ assert p_parts is not None
+ splits.add(p_parts["split"])
+
+ if any(not re.match(_split_re, split) for split in splits):
+ raise ValueError(f"Split name should match '{_split_re}'' but got '{splits}'.")
+ sorted_splits = [str(split) for split in DEFAULT_SPLITS if split in splits] + sorted(
+ splits - {str(split) for split in DEFAULT_SPLITS}
+ )
+ return {split: [split_pattern.format(split=split)] for split in sorted_splits}
+ # then check the default patterns based on train/valid/test splits
+ for patterns_dict in ALL_DEFAULT_PATTERNS:
+ non_empty_splits = []
+ for split, patterns in patterns_dict.items():
+ for pattern in patterns:
+ try:
+ data_files = pattern_resolver(pattern)
+ except FileNotFoundError:
+ continue
+ if len(data_files) > 0:
+ non_empty_splits.append(split)
+ break
+ if non_empty_splits:
+ return {split: patterns_dict[split] for split in non_empty_splits}
+ raise FileNotFoundError(f"Couldn't resolve pattern {pattern} with resolver {pattern_resolver}")
+
+
+def resolve_pattern(
+ pattern: str,
+ base_path: str,
+ allowed_extensions: Optional[list[str]] = None,
+ download_config: Optional[DownloadConfig] = None,
+) -> list[str]:
+ """
+ Resolve the paths and URLs of the data files from the pattern passed by the user.
+
+ You can use patterns to resolve multiple local files. Here are a few examples:
+ - *.csv to match all the CSV files at the first level
+ - **.csv to match all the CSV files at any level
+ - data/* to match all the files inside "data"
+ - data/** to match all the files inside "data" and its subdirectories
+
+ The patterns are resolved using the fsspec glob. In fsspec>=2023.12.0 this is equivalent to
+ Python's glob.glob, Path.glob, Path.match and fnmatch where ** is unsupported with a prefix/suffix
+ other than a forward slash /.
+
+ More generally:
+ - '*' matches any character except a forward-slash (to match just the file or directory name)
+ - '**' matches any character including a forward-slash /
+
+ Hidden files and directories (i.e. whose names start with a dot) are ignored, unless they are explicitly requested.
+ The same applies to special directories that start with a double underscore like "__pycache__".
+ You can still include one if the pattern explicitly mentions it:
+ - to include a hidden file: "*/.hidden.txt" or "*/.*"
+ - to include a hidden directory: ".hidden/*" or ".*/*"
+ - to include a special directory: "__special__/*" or "__*/*"
+
+ Example::
+
+ >>> from datasets.data_files import resolve_pattern
+ >>> base_path = "."
+ >>> resolve_pattern("docs/**/*.py", base_path)
+ [/Users/mariosasko/Desktop/projects/datasets/docs/source/_config.py']
+
+ Args:
+ pattern (str): Unix pattern or paths or URLs of the data files to resolve.
+ The paths can be absolute or relative to base_path.
+ Remote filesystems using fsspec are supported, e.g. with the hf:// protocol.
+ base_path (str): Base path to use when resolving relative paths.
+ allowed_extensions (Optional[list], optional): White-list of file extensions to use. Defaults to None (all extensions).
+ For example: allowed_extensions=[".csv", ".json", ".txt", ".parquet"]
+ download_config ([`DownloadConfig`], *optional*): Specific download configuration parameters.
+ Returns:
+ List[str]: List of paths or URLs to the local or remote files that match the patterns.
+ """
+ if is_relative_path(pattern):
+ pattern = xjoin(base_path, pattern)
+ elif is_local_path(pattern):
+ base_path = os.path.splitdrive(pattern)[0] + os.sep
+ else:
+ base_path = ""
+ pattern, storage_options = _prepare_path_and_storage_options(pattern, download_config=download_config)
+ fs, fs_pattern = url_to_fs(pattern, **storage_options)
+ files_to_ignore = set(FILES_TO_IGNORE) - {xbasename(pattern)}
+ protocol = (
+ pattern.split("://")[0]
+ if "://" in pattern
+ else (fs.protocol if isinstance(fs.protocol, str) else fs.protocol[0])
+ )
+ protocol_prefix = protocol + "://" if protocol != "file" else ""
+ glob_kwargs = {}
+ if protocol == "hf":
+ # 10 times faster glob with detail=True (ignores costly info like lastCommit)
+ glob_kwargs["expand_info"] = False
+
+ # if the pattern contains hops like "zip://csv/*.csv::data.zip", we need to keep them after globbing
+ _, *rest_hops = pattern.split("::")
+ matched_paths = []
+ for filepath, info in fs.glob(fs_pattern, detail=True, **glob_kwargs).items():
+ if not (info["type"] == "file" or (info.get("islink") and os.path.isfile(os.path.realpath(filepath)))) or (
+ xbasename(filepath) in files_to_ignore
+ ):
+ continue
+ if _is_inside_unrequested_special_dir(filepath, fs_pattern):
+ continue
+ if _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(filepath, fs_pattern):
+ continue
+ filepath = filepath if "://" in filepath else protocol_prefix + filepath
+ if rest_hops:
+ filepath = "::".join([filepath] + rest_hops)
+ matched_paths.append(filepath)
+ # ignore .ipynb and __pycache__, but keep /../
+ if allowed_extensions is not None:
+ out = [
+ filepath
+ for filepath in matched_paths
+ if any("." + suffix in allowed_extensions for suffix in xbasename(filepath).split(".")[1:])
+ ]
+ if len(out) < len(matched_paths):
+ invalid_matched_files = list(set(matched_paths) - set(out))
+ logger.info(
+ f"Some files matched the pattern '{pattern}' but don't have valid data file extensions: {invalid_matched_files}"
+ )
+ else:
+ out = matched_paths
+ if not out:
+ error_msg = f"Unable to find '{pattern}'"
+ if allowed_extensions is not None:
+ error_msg += f" with any supported extension {list(allowed_extensions)}"
+ raise FileNotFoundError(error_msg)
+ return out
+
+
+def get_data_patterns(base_path: str, download_config: Optional[DownloadConfig] = None) -> dict[str, list[str]]:
+ """
+ Get the default pattern from a directory testing all the supported patterns.
+ The first patterns to return a non-empty list of data files is returned.
+
+ Some examples of supported patterns:
+
+ Input:
+
+ my_dataset_repository/
+ ├── README.md
+ └── dataset.csv
+
+ Output:
+
+ {'train': ['**']}
+
+ Input:
+
+ my_dataset_repository/
+ ├── README.md
+ ├── train.csv
+ └── test.csv
+
+ my_dataset_repository/
+ ├── README.md
+ └── data/
+ ├── train.csv
+ └── test.csv
+
+ my_dataset_repository/
+ ├── README.md
+ ├── train_0.csv
+ ├── train_1.csv
+ ├── train_2.csv
+ ├── train_3.csv
+ ├── test_0.csv
+ └── test_1.csv
+
+ Output:
+
+ {'train': ['**/train[-._ 0-9]*', '**/*[-._ 0-9]train[-._ 0-9]*', '**/training[-._ 0-9]*', '**/*[-._ 0-9]training[-._ 0-9]*'],
+ 'test': ['**/test[-._ 0-9]*', '**/*[-._ 0-9]test[-._ 0-9]*', '**/testing[-._ 0-9]*', '**/*[-._ 0-9]testing[-._ 0-9]*', ...]}
+
+ Input:
+
+ my_dataset_repository/
+ ├── README.md
+ └── data/
+ ├── train/
+ │ ├── shard_0.csv
+ │ ├── shard_1.csv
+ │ ├── shard_2.csv
+ │ └── shard_3.csv
+ └── test/
+ ├── shard_0.csv
+ └── shard_1.csv
+
+ Output:
+
+ {'train': ['**/train/**', '**/train[-._ 0-9]*/**', '**/*[-._ 0-9]train/**', '**/*[-._ 0-9]train[-._ 0-9]*/**', ...],
+ 'test': ['**/test/**', '**/test[-._ 0-9]*/**', '**/*[-._ 0-9]test/**', '**/*[-._ 0-9]test[-._ 0-9]*/**', ...]}
+
+ Input:
+
+ my_dataset_repository/
+ ├── README.md
+ └── data/
+ ├── train-00000-of-00003.csv
+ ├── train-00001-of-00003.csv
+ ├── train-00002-of-00003.csv
+ ├── test-00000-of-00001.csv
+ ├── random-00000-of-00003.csv
+ ├── random-00001-of-00003.csv
+ └── random-00002-of-00003.csv
+
+ Output:
+
+ {'train': ['data/train-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*'],
+ 'test': ['data/test-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*'],
+ 'random': ['data/random-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*']}
+
+ In order, it first tests if SPLIT_PATTERN_SHARDED works, otherwise it tests the patterns in ALL_DEFAULT_PATTERNS.
+ """
+ resolver = partial(resolve_pattern, base_path=base_path, download_config=download_config)
+ try:
+ return _get_data_files_patterns(resolver)
+ except FileNotFoundError:
+ raise EmptyDatasetError(f"The directory at {base_path} doesn't contain any data files") from None
+
+
+def _get_single_origin_metadata(
+ data_file: str,
+ download_config: Optional[DownloadConfig] = None,
+) -> SingleOriginMetadata:
+ if data_file.startswith(config.HF_ENDPOINT):
+ fs = HfFileSystem(endpoint=config.HF_ENDPOINT, token=download_config.token)
+ data_file = "hf://" + data_file[len(config.HF_ENDPOINT) + 1 :]
+ data_file = data_file.replace("/resolve/", "/" if data_file.startswith("hf://buckets/") else "@", 1)
+ fs_path = data_file
+ else:
+ data_file, storage_options = _prepare_path_and_storage_options(data_file, download_config=download_config)
+ fs, fs_path = url_to_fs(data_file, **storage_options)
+ if isinstance(fs, HfFileSystem):
+ resolved_path = fs.resolve_path(fs_path)
+ if hasattr(resolved_path, "revision"): # no revision for buckets
+ return resolved_path.repo_id, resolved_path.revision
+ info = fs.info(fs_path)
+ # s3fs uses "ETag", gcsfs uses "etag", and for local we simply check mtime
+ for key in ["ETag", "etag", "mtime"]:
+ if key in info:
+ return (str(info[key]),)
+ return ()
+
+
+def _get_origin_metadata(
+ data_files: list[str],
+ download_config: Optional[DownloadConfig] = None,
+ max_workers: Optional[int] = None,
+) -> list[SingleOriginMetadata]:
+ max_workers = max_workers if max_workers is not None else config.HF_DATASETS_MULTITHREADING_MAX_WORKERS
+ if all("hf://" in data_file for data_file in data_files):
+ # No need for multithreading here since the origin metadata of HF files
+ # is (repo_id, revision) and is cached after first .info() call.
+ return [
+ _get_single_origin_metadata(data_file, download_config=download_config)
+ for data_file in hf_tqdm(
+ data_files,
+ desc="Resolving data files",
+ # set `disable=None` rather than `disable=False` by default to disable progress bar when no TTY attached
+ disable=len(data_files) <= 16 or None,
+ )
+ ]
+ return thread_map(
+ partial(_get_single_origin_metadata, download_config=download_config),
+ data_files,
+ max_workers=max_workers,
+ tqdm_class=hf_tqdm,
+ desc="Resolving data files",
+ # set `disable=None` rather than `disable=False` by default to disable progress bar when no TTY attached
+ disable=len(data_files) <= 16 or None,
+ )
+
+
+class DataFilesList(list[str]):
+ """
+ List of data files (absolute local paths or URLs).
+ It has two construction methods given the user's data files patterns:
+ - ``from_hf_repo``: resolve patterns inside a dataset repository
+ - ``from_local_or_remote``: resolve patterns from a local path
+
+ Moreover, DataFilesList has an additional attribute ``origin_metadata``.
+ It can store:
+ - the last modified time of local files
+ - ETag of remote files
+ - commit sha of a dataset repository
+
+ Thanks to this additional attribute, it is possible to hash the list
+ and get a different hash if and only if at least one file changed.
+ This is useful for caching Dataset objects that are obtained from a list of data files.
+ """
+
+ def __init__(self, data_files: list[str], origin_metadata: list[SingleOriginMetadata]) -> None:
+ super().__init__(data_files)
+ self.origin_metadata = origin_metadata
+
+ def __add__(self, other: "DataFilesList") -> "DataFilesList":
+ return DataFilesList([*self, *other], self.origin_metadata + other.origin_metadata)
+
+ @classmethod
+ def from_hf_repo(
+ cls,
+ patterns: list[str],
+ dataset_info: huggingface_hub.hf_api.DatasetInfo,
+ base_path: Optional[str] = None,
+ allowed_extensions: Optional[list[str]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ ) -> "DataFilesList":
+ base_path = f"hf://datasets/{dataset_info.id}@{dataset_info.sha}/{base_path or ''}".rstrip("/")
+ return cls.from_patterns(
+ patterns, base_path=base_path, allowed_extensions=allowed_extensions, download_config=download_config
+ )
+
+ @classmethod
+ def from_local_or_remote(
+ cls,
+ patterns: list[str],
+ base_path: Optional[str] = None,
+ allowed_extensions: Optional[list[str]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ ) -> "DataFilesList":
+ base_path = base_path if base_path is not None else Path().resolve().as_posix()
+ return cls.from_patterns(
+ patterns, base_path=base_path, allowed_extensions=allowed_extensions, download_config=download_config
+ )
+
+ @classmethod
+ def from_patterns(
+ cls,
+ patterns: list[str],
+ base_path: Optional[str] = None,
+ allowed_extensions: Optional[list[str]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ ) -> "DataFilesList":
+ base_path = base_path if base_path is not None else Path().resolve().as_posix()
+ data_files = []
+ for pattern in patterns:
+ try:
+ data_files.extend(
+ resolve_pattern(
+ pattern,
+ base_path=base_path,
+ allowed_extensions=allowed_extensions,
+ download_config=download_config,
+ )
+ )
+ except FileNotFoundError:
+ if not has_magic(pattern):
+ raise
+ origin_metadata = _get_origin_metadata(data_files, download_config=download_config)
+ return cls(data_files, origin_metadata)
+
+ def filter(
+ self, *, extensions: Optional[list[str]] = None, file_names: Optional[list[str]] = None
+ ) -> "DataFilesList":
+ patterns = []
+ if extensions:
+ ext_pattern = "|".join(re.escape(ext) for ext in extensions)
+ patterns.append(re.compile(f".*({ext_pattern})(\\..+)?$"))
+ if file_names:
+ fn_pattern = "|".join(re.escape(fn) for fn in file_names)
+ patterns.append(re.compile(rf".*[\/]?({fn_pattern})$"))
+ if patterns:
+ return DataFilesList(
+ [data_file for data_file in self if any(pattern.match(data_file) for pattern in patterns)],
+ origin_metadata=self.origin_metadata,
+ )
+ else:
+ return DataFilesList(list(self), origin_metadata=self.origin_metadata)
+
+
+class DataFilesDict(dict[str, DataFilesList]):
+ """
+ Dict of split_name -> list of data files (absolute local paths or URLs).
+ It has two construction methods given the user's data files patterns :
+ - ``from_hf_repo``: resolve patterns inside a dataset repository
+ - ``from_local_or_remote``: resolve patterns from a local path
+
+ Moreover, each list is a DataFilesList. It is possible to hash the dictionary
+ and get a different hash if and only if at least one file changed.
+ For more info, see [`DataFilesList`].
+
+ This is useful for caching Dataset objects that are obtained from a list of data files.
+
+ Changing the order of the keys of this dictionary also doesn't change its hash.
+ """
+
+ @classmethod
+ def from_local_or_remote(
+ cls,
+ patterns: dict[str, Union[list[str], DataFilesList]],
+ base_path: Optional[str] = None,
+ allowed_extensions: Optional[list[str]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ ) -> "DataFilesDict":
+ out = cls()
+ for key, patterns_for_key in patterns.items():
+ out[key] = (
+ patterns_for_key
+ if isinstance(patterns_for_key, DataFilesList)
+ else DataFilesList.from_local_or_remote(
+ patterns_for_key,
+ base_path=base_path,
+ allowed_extensions=allowed_extensions,
+ download_config=download_config,
+ )
+ )
+ return out
+
+ @classmethod
+ def from_hf_repo(
+ cls,
+ patterns: dict[str, Union[list[str], DataFilesList]],
+ dataset_info: huggingface_hub.hf_api.DatasetInfo,
+ base_path: Optional[str] = None,
+ allowed_extensions: Optional[list[str]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ ) -> "DataFilesDict":
+ out = cls()
+ for key, patterns_for_key in patterns.items():
+ out[key] = (
+ patterns_for_key
+ if isinstance(patterns_for_key, DataFilesList)
+ else DataFilesList.from_hf_repo(
+ patterns_for_key,
+ dataset_info=dataset_info,
+ base_path=base_path,
+ allowed_extensions=allowed_extensions,
+ download_config=download_config,
+ )
+ )
+ return out
+
+ @classmethod
+ def from_patterns(
+ cls,
+ patterns: dict[str, Union[list[str], DataFilesList]],
+ base_path: Optional[str] = None,
+ allowed_extensions: Optional[list[str]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ ) -> "DataFilesDict":
+ out = cls()
+ for key, patterns_for_key in patterns.items():
+ out[key] = (
+ patterns_for_key
+ if isinstance(patterns_for_key, DataFilesList)
+ else DataFilesList.from_patterns(
+ patterns_for_key,
+ base_path=base_path,
+ allowed_extensions=allowed_extensions,
+ download_config=download_config,
+ )
+ )
+ return out
+
+ def filter(
+ self, *, extensions: Optional[list[str]] = None, file_names: Optional[list[str]] = None
+ ) -> "DataFilesDict":
+ out = type(self)()
+ for key, data_files_list in self.items():
+ out[key] = data_files_list.filter(extensions=extensions, file_names=file_names)
+ return out
+
+
+class DataFilesPatternsList(list[str]):
+ """
+ List of data files patterns (absolute local paths or URLs).
+ For each pattern there should also be a list of allowed extensions
+ to keep, or a None ot keep all the files for the pattern.
+ """
+
+ def __init__(
+ self,
+ patterns: list[str],
+ allowed_extensions: list[Optional[list[str]]],
+ ):
+ super().__init__(patterns)
+ self.allowed_extensions = allowed_extensions
+
+ def __add__(self, other):
+ return DataFilesList([*self, *other], self.allowed_extensions + other.allowed_extensions)
+
+ @classmethod
+ def from_patterns(
+ cls, patterns: list[str], allowed_extensions: Optional[list[str]] = None
+ ) -> "DataFilesPatternsList":
+ return cls(patterns, [allowed_extensions] * len(patterns))
+
+ def resolve(
+ self,
+ base_path: str,
+ download_config: Optional[DownloadConfig] = None,
+ ) -> "DataFilesList":
+ base_path = base_path if base_path is not None else Path().resolve().as_posix()
+ data_files = []
+ for pattern, allowed_extensions in zip(self, self.allowed_extensions):
+ try:
+ data_files.extend(
+ resolve_pattern(
+ pattern,
+ base_path=base_path,
+ allowed_extensions=allowed_extensions,
+ download_config=download_config,
+ )
+ )
+ except FileNotFoundError:
+ if not has_magic(pattern):
+ raise
+ origin_metadata = _get_origin_metadata(data_files, download_config=download_config)
+ return DataFilesList(data_files, origin_metadata)
+
+ def filter_extensions(self, extensions: list[str]) -> "DataFilesPatternsList":
+ return DataFilesPatternsList(
+ self, [allowed_extensions + extensions for allowed_extensions in self.allowed_extensions]
+ )
+
+
+class DataFilesPatternsDict(dict[str, DataFilesPatternsList]):
+ """
+ Dict of split_name -> list of data files patterns (absolute local paths or URLs).
+ """
+
+ @classmethod
+ def from_patterns(
+ cls, patterns: dict[str, list[str]], allowed_extensions: Optional[list[str]] = None
+ ) -> "DataFilesPatternsDict":
+ out = cls()
+ for key, patterns_for_key in patterns.items():
+ out[key] = (
+ patterns_for_key
+ if isinstance(patterns_for_key, DataFilesPatternsList)
+ else DataFilesPatternsList.from_patterns(
+ patterns_for_key,
+ allowed_extensions=allowed_extensions,
+ )
+ )
+ return out
+
+ def resolve(
+ self,
+ base_path: str,
+ download_config: Optional[DownloadConfig] = None,
+ ) -> "DataFilesDict":
+ out = DataFilesDict()
+ for key, data_files_patterns_list in self.items():
+ out[key] = data_files_patterns_list.resolve(base_path, download_config)
+ return out
+
+ def filter_extensions(self, extensions: list[str]) -> "DataFilesPatternsDict":
+ out = type(self)()
+ for key, data_files_patterns_list in self.items():
+ out[key] = data_files_patterns_list.filter_extensions(extensions)
+ return out
diff --git a/venv/Lib/site-packages/datasets/dataset_dict.py b/venv/Lib/site-packages/datasets/dataset_dict.py
new file mode 100644
index 0000000000000000000000000000000000000000..4abea0a381a1ff83e29e19f138d6f6f48e5139bb
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/dataset_dict.py
@@ -0,0 +1,2774 @@
+import contextlib
+import copy
+import itertools
+import json
+import math
+import posixpath
+import random
+import re
+import time
+from collections.abc import Sequence
+from functools import partial
+from typing import Callable, Literal, Optional, Union
+
+import fsspec
+import numpy as np
+from fsspec.core import url_to_fs
+from fsspec.implementations.dirfs import DirFileSystem
+from huggingface_hub import (
+ CommitInfo,
+ CommitOperationAdd,
+ CommitOperationDelete,
+ HfApi,
+ HfFileSystem,
+ HfFileSystemResolvedPath,
+)
+from huggingface_hub.utils import EntryNotFoundError, HfHubHTTPError, RepositoryNotFoundError
+from packaging import version
+
+from . import __version__, config
+from .arrow_dataset import (
+ Dataset,
+ _get_updated_dataset_card,
+)
+from .features import Features
+from .features.features import FeatureType
+from .iterable_dataset import IterableDataset
+from .naming import _split_re
+from .splits import NamedSplit, Split, SplitInfo
+from .table import Table
+from .utils import logging
+from .utils.doc_utils import is_documented_by
+from .utils.typing import PathLike
+
+
+if config.HF_HUB_VERSION >= version.parse("1.6.0"):
+ from huggingface_hub.errors import BucketNotFoundError
+ from huggingface_hub.hf_file_system import HfFileSystemResolvedBucketPath, HfFileSystemResolvedRepositoryPath
+
+else:
+ BucketNotFoundError = None
+ HfFileSystemResolvedBucketPath = None
+ HfFileSystemResolvedRepositoryPath = HfFileSystemResolvedPath
+
+
+logger = logging.get_logger(__name__)
+
+
+class bind(partial):
+ def __call__(self, *fn_args, **fn_kwargs):
+ return self.func(*fn_args, *self.args, **fn_kwargs)
+
+
+class DatasetDict(dict[Union[str, NamedSplit], "Dataset"]):
+ """A dictionary (dict of str: datasets.Dataset) with dataset transforms methods (map, filter, etc.)"""
+
+ def _check_values_type(self):
+ for dataset in self.values():
+ if not isinstance(dataset, Dataset):
+ raise TypeError(f"Values in `DatasetDict` should be of type `Dataset` but got type '{type(dataset)}'")
+
+ def _check_values_features(self):
+ items = list(self.items())
+ for item_a, item_b in zip(items[:-1], items[1:]):
+ if item_a[1].features != item_b[1].features:
+ raise ValueError(
+ f"All datasets in `DatasetDict` should have the same features but features for '{item_a[0]}' and '{item_b[0]}' don't match: {item_a[1].features} != {item_b[1].features}"
+ )
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ # Here `del` is used to del the pyarrow tables. This properly closes the files used for memory mapped tables
+ for dataset in self.values():
+ if hasattr(dataset, "_data"):
+ del dataset._data
+ if hasattr(dataset, "_indices"):
+ del dataset._indices
+
+ def __getitem__(self, k) -> Dataset:
+ if isinstance(k, (str, NamedSplit)) or len(self) == 0:
+ return super().__getitem__(k)
+ else:
+ available_suggested_splits = [
+ split for split in (Split.TRAIN, Split.TEST, Split.VALIDATION) if split in self
+ ]
+ suggested_split = available_suggested_splits[0] if available_suggested_splits else list(self)[0]
+ raise KeyError(
+ f"Invalid key: {k}. Please first select a split. For example: "
+ f"`my_dataset_dictionary['{suggested_split}'][{k}]`. "
+ f"Available splits: {sorted(self)}"
+ )
+
+ @property
+ def data(self) -> dict[str, Table]:
+ """The Apache Arrow tables backing each split.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds.data
+ ```
+ """
+ self._check_values_type()
+ return {k: dataset.data for k, dataset in self.items()}
+
+ @property
+ def cache_files(self) -> dict[str, dict]:
+ """The cache files containing the Apache Arrow table backing each split.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds.cache_files
+ {'test': [{'filename': '/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/40d411e45a6ce3484deed7cc15b82a53dad9a72aafd9f86f8f227134bec5ca46/rotten_tomatoes_movie_review-test.arrow'}],
+ 'train': [{'filename': '/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/40d411e45a6ce3484deed7cc15b82a53dad9a72aafd9f86f8f227134bec5ca46/rotten_tomatoes_movie_review-train.arrow'}],
+ 'validation': [{'filename': '/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/40d411e45a6ce3484deed7cc15b82a53dad9a72aafd9f86f8f227134bec5ca46/rotten_tomatoes_movie_review-validation.arrow'}]}
+ ```
+ """
+ self._check_values_type()
+ return {k: dataset.cache_files for k, dataset in self.items()}
+
+ @property
+ def num_columns(self) -> dict[str, int]:
+ """Number of columns in each split of the dataset.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds.num_columns
+ {'test': 2, 'train': 2, 'validation': 2}
+ ```
+ """
+ self._check_values_type()
+ return {k: dataset.num_columns for k, dataset in self.items()}
+
+ @property
+ def num_rows(self) -> dict[str, int]:
+ """Number of rows in each split of the dataset.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds.num_rows
+ {'test': 1066, 'train': 8530, 'validation': 1066}
+ ```
+ """
+ self._check_values_type()
+ return {k: dataset.num_rows for k, dataset in self.items()}
+
+ @property
+ def column_names(self) -> dict[str, list[str]]:
+ """Names of the columns in each split of the dataset.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds.column_names
+ {'test': ['text', 'label'],
+ 'train': ['text', 'label'],
+ 'validation': ['text', 'label']}
+ ```
+ """
+ self._check_values_type()
+ return {k: dataset.column_names for k, dataset in self.items()}
+
+ @property
+ def shape(self) -> dict[str, tuple[int]]:
+ """Shape of each split of the dataset (number of rows, number of columns).
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds.shape
+ {'test': (1066, 2), 'train': (8530, 2), 'validation': (1066, 2)}
+ ```
+ """
+ self._check_values_type()
+ return {k: dataset.shape for k, dataset in self.items()}
+
+ def flatten(self, max_depth=16) -> "DatasetDict":
+ """Flatten the Apache Arrow Table of each split (nested features are flatten).
+ Each column with a struct type is flattened into one column per struct field.
+ Other columns are left unchanged.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("rajpurkar/squad")
+ >>> ds["train"].features
+ {'id': Value('string'),
+ 'title': Value('string'),
+ 'context': Value('string'),
+ 'question': Value('string'),
+ 'answers.text': List(Value('string')),
+ 'answers.answer_start': List(Value('int32'))}
+ >>> ds.flatten()
+ DatasetDict({
+ train: Dataset({
+ features: ['id', 'title', 'context', 'question', 'answers.text', 'answers.answer_start'],
+ num_rows: 87599
+ })
+ validation: Dataset({
+ features: ['id', 'title', 'context', 'question', 'answers.text', 'answers.answer_start'],
+ num_rows: 10570
+ })
+ })
+ ```
+ """
+ self._check_values_type()
+ return DatasetDict({k: dataset.flatten(max_depth=max_depth) for k, dataset in self.items()})
+
+ def unique(self, column: str) -> dict[str, list]:
+ """Return a list of the unique elements in a column for each split.
+
+ This is implemented in the low-level backend and as such, very fast.
+
+ Args:
+ column (`str`):
+ column name (list all the column names with [`~datasets.DatasetDict.column_names`])
+
+ Returns:
+ Dict[`str`, `list`]: Dictionary of unique elements in the given column.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds.unique("label")
+ {'test': [1, 0], 'train': [1, 0], 'validation': [1, 0]}
+ ```
+ """
+ self._check_values_type()
+ return {k: dataset.unique(column) for k, dataset in self.items()}
+
+ def cleanup_cache_files(self) -> dict[str, int]:
+ """Clean up all cache files in the dataset cache directory, excepted the currently used cache file if there is one.
+ Be careful when running this command that no other process is currently using other cache files.
+
+ Return:
+ `Dict` with the number of removed files for each split
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds.cleanup_cache_files()
+ {'test': 0, 'train': 0, 'validation': 0}
+ ```
+ """
+ self._check_values_type()
+ return {k: dataset.cleanup_cache_files() for k, dataset in self.items()}
+
+ def __repr__(self):
+ repr = "\n".join([f"{k}: {v}" for k, v in self.items()])
+ repr = re.sub(r"^", " " * 4, repr, count=0, flags=re.M)
+ return f"DatasetDict({{\n{repr}\n}})"
+
+ def cast(self, features: Features) -> "DatasetDict":
+ """
+ Cast the dataset to a new set of features.
+ The transformation is applied to all the datasets of the dataset dictionary.
+
+ Args:
+ features ([`Features`]):
+ New features to cast the dataset to.
+ The name and order of the fields in the features must match the current column names.
+ The type of the data must also be convertible from one type to the other.
+ For non-trivial conversion, e.g. `string` <-> `ClassLabel` you should use [`~DatasetDict.map`] to update the dataset.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset, ClassLabel, Value
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds["train"].features
+ {'label': ClassLabel(names=['neg', 'pos']),
+ 'text': Value('string')}
+ >>> new_features = ds["train"].features.copy()
+ >>> new_features['label'] = ClassLabel(names=['bad', 'good'])
+ >>> new_features['text'] = Value('large_string')
+ >>> ds = ds.cast(new_features)
+ >>> ds["train"].features
+ {'label': ClassLabel(names=['bad', 'good']),
+ 'text': Value('large_string')}
+ ```
+ """
+ self._check_values_type()
+ return DatasetDict({k: dataset.cast(features=features) for k, dataset in self.items()})
+
+ def cast_column(self, column: str, feature) -> "DatasetDict":
+ """Cast column to feature for decoding.
+
+ Args:
+ column (`str`):
+ Column name.
+ feature ([`Feature`]):
+ Target feature.
+
+ Returns:
+ [`DatasetDict`]
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset, ClassLabel
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds["train"].features
+ {'label': ClassLabel(names=['neg', 'pos']),
+ 'text': Value('string')}
+ >>> ds = ds.cast_column('label', ClassLabel(names=['bad', 'good']))
+ >>> ds["train"].features
+ {'label': ClassLabel(names=['bad', 'good']),
+ 'text': Value('string')}
+ ```
+ """
+ self._check_values_type()
+ return DatasetDict({k: dataset.cast_column(column=column, feature=feature) for k, dataset in self.items()})
+
+ def remove_columns(self, column_names: Union[str, list[str]]) -> "DatasetDict":
+ """
+ Remove one or several column(s) from each split in the dataset
+ and the features associated to the column(s).
+
+ The transformation is applied to all the splits of the dataset dictionary.
+
+ You can also remove a column using [`~DatasetDict.map`] with `remove_columns` but the present method
+ doesn't copy the data of the remaining columns and is thus faster.
+
+ Args:
+ column_names (`Union[str, list[str]]`):
+ Name of the column(s) to remove.
+
+ Returns:
+ [`DatasetDict`]: A copy of the dataset object without the columns to remove.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds = ds.remove_columns("label")
+ DatasetDict({
+ train: Dataset({
+ features: ['text'],
+ num_rows: 8530
+ })
+ validation: Dataset({
+ features: ['text'],
+ num_rows: 1066
+ })
+ test: Dataset({
+ features: ['text'],
+ num_rows: 1066
+ })
+ })
+ ```
+ """
+ self._check_values_type()
+ return DatasetDict({k: dataset.remove_columns(column_names=column_names) for k, dataset in self.items()})
+
+ def rename_column(self, original_column_name: str, new_column_name: str) -> "DatasetDict":
+ """
+ Rename a column in the dataset and move the features associated to the original column under the new column name.
+ The transformation is applied to all the datasets of the dataset dictionary.
+
+ You can also rename a column using [`~DatasetDict.map`] with `remove_columns` but the present method:
+ - takes care of moving the original features under the new column name.
+ - doesn't copy the data to a new dataset and is thus much faster.
+
+ Args:
+ original_column_name (`str`):
+ Name of the column to rename.
+ new_column_name (`str`):
+ New name for the column.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds = ds.rename_column("label", "label_new")
+ DatasetDict({
+ train: Dataset({
+ features: ['text', 'label_new'],
+ num_rows: 8530
+ })
+ validation: Dataset({
+ features: ['text', 'label_new'],
+ num_rows: 1066
+ })
+ test: Dataset({
+ features: ['text', 'label_new'],
+ num_rows: 1066
+ })
+ })
+ ```
+ """
+ self._check_values_type()
+ return DatasetDict(
+ {
+ k: dataset.rename_column(
+ original_column_name=original_column_name,
+ new_column_name=new_column_name,
+ )
+ for k, dataset in self.items()
+ }
+ )
+
+ def rename_columns(self, column_mapping: dict[str, str]) -> "DatasetDict":
+ """
+ Rename several columns in the dataset, and move the features associated to the original columns under
+ the new column names.
+ The transformation is applied to all the datasets of the dataset dictionary.
+
+ Args:
+ column_mapping (`Dict[str, str]`):
+ A mapping of columns to rename to their new names.
+
+ Returns:
+ [`DatasetDict`]: A copy of the dataset with renamed columns.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds.rename_columns({'text': 'text_new', 'label': 'label_new'})
+ DatasetDict({
+ train: Dataset({
+ features: ['text_new', 'label_new'],
+ num_rows: 8530
+ })
+ validation: Dataset({
+ features: ['text_new', 'label_new'],
+ num_rows: 1066
+ })
+ test: Dataset({
+ features: ['text_new', 'label_new'],
+ num_rows: 1066
+ })
+ })
+ ```
+ """
+ self._check_values_type()
+ return DatasetDict({k: dataset.rename_columns(column_mapping=column_mapping) for k, dataset in self.items()})
+
+ def select_columns(self, column_names: Union[str, list[str]]) -> "DatasetDict":
+ """Select one or several column(s) from each split in the dataset and
+ the features associated to the column(s).
+
+ The transformation is applied to all the splits of the dataset
+ dictionary.
+
+ Args:
+ column_names (`Union[str, list[str]]`):
+ Name of the column(s) to keep.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds.select_columns("text")
+ DatasetDict({
+ train: Dataset({
+ features: ['text'],
+ num_rows: 8530
+ })
+ validation: Dataset({
+ features: ['text'],
+ num_rows: 1066
+ })
+ test: Dataset({
+ features: ['text'],
+ num_rows: 1066
+ })
+ })
+ ```
+ """
+ self._check_values_type()
+ return DatasetDict({k: dataset.select_columns(column_names=column_names) for k, dataset in self.items()})
+
+ def class_encode_column(self, column: str, include_nulls: bool = False) -> "DatasetDict":
+ """Casts the given column as [`~datasets.features.ClassLabel`] and updates the tables.
+
+ Args:
+ column (`str`):
+ The name of the column to cast.
+ include_nulls (`bool`, defaults to `False`):
+ Whether to include null values in the class labels. If `True`, the null values will be encoded as the `"None"` class label.
+
+
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("google/boolq")
+ >>> ds["train"].features
+ {'answer': Value('bool'),
+ 'passage': Value('string'),
+ 'question': Value('string')}
+ >>> ds = ds.class_encode_column("answer")
+ >>> ds["train"].features
+ {'answer': ClassLabel(num_classes=2, names=['False', 'True']),
+ 'passage': Value('string'),
+ 'question': Value('string')}
+ ```
+ """
+ self._check_values_type()
+ return DatasetDict(
+ {k: dataset.class_encode_column(column=column, include_nulls=include_nulls) for k, dataset in self.items()}
+ )
+
+ @contextlib.contextmanager
+ def formatted_as(
+ self,
+ type: Optional[str] = None,
+ columns: Optional[list] = None,
+ output_all_columns: bool = False,
+ **format_kwargs,
+ ):
+ """To be used in a `with` statement. Set `__getitem__` return format (type and columns).
+ The transformation is applied to all the datasets of the dataset dictionary.
+
+ Args:
+ type (`str`, *optional*):
+ Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`.
+ `None` means `__getitem__` returns python objects (default).
+ columns (`list[str]`, *optional*):
+ Columns to format in the output.
+ `None` means `__getitem__` returns all columns (default).
+ output_all_columns (`bool`, defaults to False):
+ Keep un-formatted columns as well in the output (as python objects).
+ **format_kwargs (additional keyword arguments):
+ Keywords arguments passed to the convert function like `np.array`, `torch.tensor` or `tensorflow.ragged.constant`.
+ """
+ self._check_values_type()
+ old_format_type = {k: dataset._format_type for k, dataset in self.items()}
+ old_format_kwargs = {k: dataset._format_kwargs for k, dataset in self.items()}
+ old_format_columns = {k: dataset._format_columns for k, dataset in self.items()}
+ old_output_all_columns = {k: dataset._output_all_columns for k, dataset in self.items()}
+ try:
+ self.set_format(type, columns, output_all_columns, **format_kwargs)
+ yield
+ finally:
+ for k, dataset in self.items():
+ dataset.set_format(
+ old_format_type[k],
+ old_format_columns[k],
+ old_output_all_columns[k],
+ **old_format_kwargs[k],
+ )
+
+ def set_format(
+ self,
+ type: Optional[str] = None,
+ columns: Optional[list] = None,
+ output_all_columns: bool = False,
+ **format_kwargs,
+ ):
+ """Set `__getitem__` return format (type and columns).
+ The format is set for every dataset in the dataset dictionary.
+
+ Args:
+ type (`str`, *optional*):
+ Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`.
+ `None` means `__getitem__` returns python objects (default).
+ columns (`list[str]`, *optional*):
+ Columns to format in the output.
+ `None` means `__getitem__` returns all columns (default).
+ output_all_columns (`bool`, defaults to False):
+ Keep un-formatted columns as well in the output (as python objects),
+ **format_kwargs (additional keyword arguments):
+ Keywords arguments passed to the convert function like `np.array`, `torch.tensor` or `tensorflow.ragged.constant`.
+
+ It is possible to call `map` after calling `set_format`. Since `map` may add new columns, then the list of formatted columns
+ gets updated. In this case, if you apply `map` on a dataset to add a new column, then this column will be formatted:
+
+ `new formatted columns = (all columns - previously unformatted columns)`
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> from transformers import AutoTokenizer
+ >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+ >>> ds = ds.map(lambda x: tokenizer(x["text"], truncation=True, padding=True), batched=True)
+ >>> ds.set_format(type="numpy", columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])
+ >>> ds["train"].format
+ {'columns': ['input_ids', 'token_type_ids', 'attention_mask', 'label'],
+ 'format_kwargs': {},
+ 'output_all_columns': False,
+ 'type': 'numpy'}
+ ```
+ """
+ self._check_values_type()
+ for dataset in self.values():
+ dataset.set_format(
+ type=type,
+ columns=columns,
+ output_all_columns=output_all_columns,
+ **format_kwargs,
+ )
+
+ def reset_format(self):
+ """Reset `__getitem__` return format to python objects and all columns.
+ The transformation is applied to all the datasets of the dataset dictionary.
+
+ Same as `self.set_format()`
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> from transformers import AutoTokenizer
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+ >>> ds = ds.map(lambda x: tokenizer(x["text"], truncation=True, padding=True), batched=True)
+ >>> ds.set_format(type="numpy", columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'])
+ >>> ds["train"].format
+ {'columns': ['input_ids', 'token_type_ids', 'attention_mask', 'label'],
+ 'format_kwargs': {},
+ 'output_all_columns': False,
+ 'type': 'numpy'}
+ >>> ds.reset_format()
+ >>> ds["train"].format
+ {'columns': ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'],
+ 'format_kwargs': {},
+ 'output_all_columns': False,
+ 'type': None}
+ ```
+ """
+ self._check_values_type()
+ for dataset in self.values():
+ dataset.set_format()
+
+ def set_transform(
+ self,
+ transform: Optional[Callable],
+ columns: Optional[list] = None,
+ output_all_columns: bool = False,
+ ):
+ """Set ``__getitem__`` return format using this transform. The transform is applied on-the-fly on batches when ``__getitem__`` is called.
+ The transform is set for every dataset in the dataset dictionary
+ As :func:`datasets.Dataset.set_format`, this can be reset using :func:`datasets.Dataset.reset_format`
+
+ Args:
+ transform (`Callable`, optional): user-defined formatting transform, replaces the format defined by :func:`datasets.Dataset.set_format`
+ A formatting function is a callable that takes a batch (as a dict) as input and returns a batch.
+ This function is applied right before returning the objects in ``__getitem__``.
+ columns (`list[str]`, optional): columns to format in the output
+ If specified, then the input batch of the transform only contains those columns.
+ output_all_columns (`bool`, default to False): keep un-formatted columns as well in the output (as python objects)
+ If set to True, then the other un-formatted columns are kept with the output of the transform.
+
+ """
+ self._check_values_type()
+ for dataset in self.values():
+ dataset.set_format(
+ "custom",
+ columns=columns,
+ output_all_columns=output_all_columns,
+ transform=transform,
+ )
+
+ def with_format(
+ self,
+ type: Optional[str] = None,
+ columns: Optional[list] = None,
+ output_all_columns: bool = False,
+ **format_kwargs,
+ ) -> "DatasetDict":
+ """Set `__getitem__` return format (type and columns). The data formatting is applied on-the-fly.
+ The format `type` (for example "numpy") is used to format batches when using `__getitem__`.
+ The format is set for every dataset in the dataset dictionary.
+
+ It's also possible to use custom transforms for formatting using [`~datasets.Dataset.with_transform`].
+
+ Contrary to [`~datasets.DatasetDict.set_format`], `with_format` returns a new [`DatasetDict`] object with new [`Dataset`] objects.
+
+ Args:
+ type (`str`, *optional*):
+ Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`.
+ `None` means `__getitem__` returns python objects (default).
+ columns (`list[str]`, *optional*):
+ Columns to format in the output.
+ `None` means `__getitem__` returns all columns (default).
+ output_all_columns (`bool`, defaults to `False`):
+ Keep un-formatted columns as well in the output (as python objects).
+ **format_kwargs (additional keyword arguments):
+ Keywords arguments passed to the convert function like `np.array`, `torch.tensor` or `tensorflow.ragged.constant`.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> from transformers import AutoTokenizer
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+ >>> ds = ds.map(lambda x: tokenizer(x['text'], truncation=True, padding=True), batched=True)
+ >>> ds["train"].format
+ {'columns': ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'],
+ 'format_kwargs': {},
+ 'output_all_columns': False,
+ 'type': None}
+ >>> ds = ds.with_format("torch")
+ >>> ds["train"].format
+ {'columns': ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'],
+ 'format_kwargs': {},
+ 'output_all_columns': False,
+ 'type': 'torch'}
+ >>> ds["train"][0]
+ {'text': 'compassionately explores the seemingly irreconcilable situation between conservative christian parents and their estranged gay and lesbian children .',
+ 'label': tensor(1),
+ 'input_ids': tensor([ 101, 18027, 16310, 16001, 1103, 9321, 178, 11604, 7235, 6617,
+ 1742, 2165, 2820, 1206, 6588, 22572, 12937, 1811, 2153, 1105,
+ 1147, 12890, 19587, 6463, 1105, 15026, 1482, 119, 102, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0]),
+ 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
+ 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])}
+ ```
+ """
+ dataset = copy.deepcopy(self)
+ dataset.set_format(
+ type=type,
+ columns=columns,
+ output_all_columns=output_all_columns,
+ **format_kwargs,
+ )
+ return dataset
+
+ def with_transform(
+ self,
+ transform: Optional[Callable],
+ columns: Optional[list] = None,
+ output_all_columns: bool = False,
+ ) -> "DatasetDict":
+ """Set `__getitem__` return format using this transform. The transform is applied on-the-fly on batches when `__getitem__` is called.
+ The transform is set for every dataset in the dataset dictionary
+
+ As [`~datasets.Dataset.set_format`], this can be reset using [`~datasets.Dataset.reset_format`].
+
+ Contrary to [`~datasets.DatasetDict.set_transform`], `with_transform` returns a new [`DatasetDict`] object with new [`Dataset`] objects.
+
+ Args:
+ transform (`Callable`, *optional*):
+ User-defined formatting transform, replaces the format defined by [`~datasets.Dataset.set_format`].
+ A formatting function is a callable that takes a batch (as a dict) as input and returns a batch.
+ This function is applied right before returning the objects in `__getitem__`.
+ columns (`list[str]`, *optional*):
+ Columns to format in the output.
+ If specified, then the input batch of the transform only contains those columns.
+ output_all_columns (`bool`, defaults to False):
+ Keep un-formatted columns as well in the output (as python objects).
+ If set to `True`, then the other un-formatted columns are kept with the output of the transform.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> from transformers import AutoTokenizer
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+ >>> def encode(example):
+ ... return tokenizer(example['text'], truncation=True, padding=True, return_tensors="pt")
+ >>> ds = ds.with_transform(encode)
+ >>> ds["train"][0]
+ {'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1]),
+ 'input_ids': tensor([ 101, 1103, 2067, 1110, 17348, 1106, 1129, 1103, 6880, 1432,
+ 112, 188, 1207, 107, 14255, 1389, 107, 1105, 1115, 1119,
+ 112, 188, 1280, 1106, 1294, 170, 24194, 1256, 3407, 1190,
+ 170, 11791, 5253, 188, 1732, 7200, 10947, 12606, 2895, 117,
+ 179, 7766, 118, 172, 15554, 1181, 3498, 6961, 3263, 1137,
+ 188, 1566, 7912, 14516, 6997, 119, 102]),
+ 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0])}
+ ```
+ """
+ dataset = copy.deepcopy(self)
+ dataset.set_transform(transform=transform, columns=columns, output_all_columns=output_all_columns)
+ return dataset
+
+ def map(
+ self,
+ function: Optional[Callable] = None,
+ with_indices: bool = False,
+ with_rank: bool = False,
+ with_split: bool = False,
+ input_columns: Optional[Union[str, list[str]]] = None,
+ batched: bool = False,
+ batch_size: Optional[int] = 1000,
+ drop_last_batch: bool = False,
+ remove_columns: Optional[Union[str, list[str]]] = None,
+ keep_in_memory: bool = False,
+ load_from_cache_file: Optional[bool] = None,
+ cache_file_names: Optional[dict[str, Optional[str]]] = None,
+ writer_batch_size: Optional[int] = 1000,
+ features: Optional[Features] = None,
+ disable_nullable: bool = False,
+ fn_kwargs: Optional[dict] = None,
+ num_proc: Optional[int] = None,
+ desc: Optional[str] = None,
+ try_original_type: Optional[bool] = True,
+ on_mixed_types: Optional[Literal["use_json"]] = None,
+ ) -> "DatasetDict":
+ """
+ Apply a function to all the examples in the table (individually or in batches) and update the table.
+ If your function returns a column that already exists, then it overwrites it.
+ The transformation is applied to all the datasets of the dataset dictionary.
+
+ You can specify whether the function should be batched or not with the `batched` parameter:
+
+ - If batched is `False`, then the function takes 1 example in and should return 1 example.
+ An example is a dictionary, e.g. `{"text": "Hello there !"}`.
+ - If batched is `True` and `batch_size` is 1, then the function takes a batch of 1 example as input and can return a batch with 1 or more examples.
+ A batch is a dictionary, e.g. a batch of 1 example is `{"text": ["Hello there !"]}`.
+ - If batched is `True` and `batch_size` is `n > 1`, then the function takes a batch of `n` examples as input and can return a batch with `n` examples, or with an arbitrary number of examples.
+ Note that the last batch may have less than `n` examples.
+ A batch is a dictionary, e.g. a batch of `n` examples is `{"text": ["Hello there !"] * n}`.
+
+ If the function is asynchronous, then `map` will run your function in parallel, with up to one thousand simultaneous calls.
+ It is recommended to use a `asyncio.Semaphore` in your function if you want to set a maximum number of operations that can run at the same time.
+
+ Args:
+ function (`callable`): with one of the following signature:
+ - `function(example: Dict[str, Any]) -> Dict[str, Any]` if `batched=False` and `with_indices=False`
+ - `function(example: Dict[str, Any], indices: int) -> Dict[str, Any]` if `batched=False` and `with_indices=True`
+ - `function(batch: Dict[str, list]) -> Dict[str, list]` if `batched=True` and `with_indices=False`
+ - `function(batch: Dict[str, list], indices: list[int]) -> Dict[str, list]` if `batched=True` and `with_indices=True`
+
+ For advanced usage, the function can also return a `pyarrow.Table`.
+ If the function is asynchronous, then `map` will run your function in parallel.
+ Moreover if your function returns nothing (`None`), then `map` will run your function and return the dataset unchanged.
+ If no function is provided, default to identity function: `lambda x: x`.
+ with_indices (`bool`, defaults to `False`):
+ Provide example indices to `function`. Note that in this case the signature of `function` should be `def function(example, idx): ...`.
+ with_rank (`bool`, defaults to `False`):
+ Provide process rank to `function`. Note that in this case the
+ signature of `function` should be `def function(example[, idx], rank): ...`.
+ with_split (`bool`, defaults to `False`):
+ Provide process split to `function`. Note that in this case the
+ signature of `function` should be `def function(example[, idx], split): ...`.
+ input_columns (`[Union[str, list[str]]]`, *optional*, defaults to `None`):
+ The columns to be passed into `function` as
+ positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument.
+ batched (`bool`, defaults to `False`):
+ Provide batch of examples to `function`.
+ batch_size (`int`, *optional*, defaults to `1000`):
+ Number of examples per batch provided to `function` if `batched=True`,
+ `batch_size <= 0` or `batch_size == None` then provide the full dataset as a single batch to `function`.
+ drop_last_batch (`bool`, defaults to `False`):
+ Whether a last batch smaller than the batch_size should be
+ dropped instead of being processed by the function.
+ remove_columns (`[Union[str, list[str]]]`, *optional*, defaults to `None`):
+ Remove a selection of columns while doing the mapping.
+ Columns will be removed before updating the examples with the output of `function`, i.e. if `function` is adding
+ columns with names in `remove_columns`, these columns will be kept.
+ keep_in_memory (`bool`, defaults to `False`):
+ Keep the dataset in memory instead of writing it to a cache file.
+ load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled):
+ If a cache file storing the current computation from `function`
+ can be identified, use it instead of recomputing.
+ cache_file_names (`[Dict[str, str]]`, *optional*, defaults to `None`):
+ Provide the name of a path for the cache file. It is used to store the
+ results of the computation instead of the automatically generated cache file name.
+ You have to provide one `cache_file_name` per dataset in the dataset dictionary.
+ writer_batch_size (`int`, default `1000`):
+ Number of rows per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`.
+ features (`[datasets.Features]`, *optional*, defaults to `None`):
+ Use a specific [`Features`] to store the cache file
+ instead of the automatically generated one.
+ disable_nullable (`bool`, defaults to `False`):
+ Disallow null values in the table.
+ fn_kwargs (`Dict`, *optional*, defaults to `None`):
+ Keyword arguments to be passed to `function`
+ num_proc (`int`, *optional*, defaults to `None`):
+ The number of processes to use for multiprocessing.
+ - If `None` or `0`, no multiprocessing is used and the operation runs in the main process.
+ - If greater than `1`, one or multiple worker processes are used to process data in parallel.
+ Note: The function passed to `map()` must be picklable for multiprocessing to work correctly
+ (i.e., prefer functions defined at the top level of a module, not inside another function or class).
+ desc (`str`, *optional*, defaults to `None`):
+ Meaningful description to be displayed alongside with the progress bar while mapping examples.
+ try_original_type (`Optional[bool]`, defaults to `True`):
+ Try to keep the types of the original columns (e.g. int32 -> int32).
+ Set to False if you want to always infer new types.
+ on_mixed_types (`Literal["use_json"]`, *optional*, defaults to `None`):
+ If "use_json", use the Json() type for mixed-types fields,
+ i.e. unstructured fields that contain data without a predefined schema.
+ In this case, a field with mixed type is set to Json().
+
+ This allow loading lists with a mix of strings/integers/floats
+ for example, or dictionaries with arbitrary value types.
+
+
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> def add_prefix(example):
+ ... example["text"] = "Review: " + example["text"]
+ ... return example
+ >>> ds = ds.map(add_prefix)
+ >>> ds["train"][0:3]["text"]
+ ['Review: the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .',
+ 'Review: the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .',
+ 'Review: effective but too-tepid biopic']
+
+ # process a batch of examples
+ >>> ds = ds.map(lambda example: tokenizer(example["text"]), batched=True)
+ # set number of processors
+ >>> ds = ds.map(add_prefix, num_proc=4)
+ ```
+ """
+ self._check_values_type()
+ if cache_file_names is None:
+ cache_file_names = dict.fromkeys(self)
+
+ dataset_dict = {}
+ for split, dataset in self.items():
+ if with_split:
+ function = bind(function, split)
+
+ dataset_dict[split] = dataset.map(
+ function=function,
+ with_indices=with_indices,
+ with_rank=with_rank,
+ input_columns=input_columns,
+ batched=batched,
+ batch_size=batch_size,
+ drop_last_batch=drop_last_batch,
+ remove_columns=remove_columns,
+ keep_in_memory=keep_in_memory,
+ load_from_cache_file=load_from_cache_file,
+ cache_file_name=cache_file_names[split],
+ writer_batch_size=writer_batch_size,
+ features=features,
+ disable_nullable=disable_nullable,
+ fn_kwargs=fn_kwargs,
+ num_proc=num_proc,
+ desc=desc,
+ try_original_type=try_original_type,
+ on_mixed_types=on_mixed_types,
+ )
+
+ if with_split:
+ function = function.func
+
+ return DatasetDict(dataset_dict)
+
+ def filter(
+ self,
+ function: Optional[Callable] = None,
+ with_indices: bool = False,
+ with_rank: bool = False,
+ input_columns: Optional[Union[str, list[str]]] = None,
+ batched: bool = False,
+ batch_size: Optional[int] = 1000,
+ keep_in_memory: bool = False,
+ load_from_cache_file: Optional[bool] = None,
+ cache_file_names: Optional[dict[str, Optional[str]]] = None,
+ writer_batch_size: Optional[int] = 1000,
+ fn_kwargs: Optional[dict] = None,
+ num_proc: Optional[int] = None,
+ desc: Optional[str] = None,
+ ) -> "DatasetDict":
+ """Apply a filter function to all the elements in the table in batches
+ and update the table so that the dataset only includes examples according to the filter function.
+ The transformation is applied to all the datasets of the dataset dictionary.
+
+ Args:
+ function (`Callable`): Callable with one of the following signatures:
+
+ - `function(example: Dict[str, Any]) -> bool` if `batched=False` and `with_indices=False` and `with_rank=False`
+ - `function(example: Dict[str, Any], *extra_args) -> bool` if `batched=False` and `with_indices=True` and/or `with_rank=True` (one extra arg for each)
+ - `function(batch: Dict[str, list]) -> list[bool]` if `batched=True` and `with_indices=False` and `with_rank=False`
+ - `function(batch: Dict[str, list], *extra_args) -> list[bool]` if `batched=True` and `with_indices=True` and/or `with_rank=True` (one extra arg for each)
+
+ If no function is provided, defaults to an always `True` function: `lambda x: True`.
+ with_indices (`bool`, defaults to `False`):
+ Provide example indices to `function`. Note that in this case the
+ signature of `function` should be `def function(example, idx[, rank]): ...`.
+ with_rank (`bool`, defaults to `False`):
+ Provide process rank to `function`. Note that in this case the
+ signature of `function` should be `def function(example[, idx], rank): ...`.
+ input_columns (`[Union[str, list[str]]]`, *optional*, defaults to `None`):
+ The columns to be passed into `function` as
+ positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument.
+ batched (`bool`, defaults to `False`):
+ Provide batch of examples to `function`.
+ batch_size (`int`, *optional*, defaults to `1000`):
+ Number of examples per batch provided to `function` if `batched=True`
+ `batch_size <= 0` or `batch_size == None` then provide the full dataset as a single batch to `function`.
+ keep_in_memory (`bool`, defaults to `False`):
+ Keep the dataset in memory instead of writing it to a cache file.
+ load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled):
+ If a cache file storing the current computation from `function`
+ can be identified, use it instead of recomputing.
+ cache_file_names (`[Dict[str, str]]`, *optional*, defaults to `None`):
+ Provide the name of a path for the cache file. It is used to store the
+ results of the computation instead of the automatically generated cache file name.
+ You have to provide one `cache_file_name` per dataset in the dataset dictionary.
+ writer_batch_size (`int`, defaults to `1000`):
+ Number of rows per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`.
+ fn_kwargs (`Dict`, *optional*, defaults to `None`):
+ Keyword arguments to be passed to `function`
+ num_proc (`int`, *optional*, defaults to `None`):
+ The number of processes to use for multiprocessing.
+ - If `None` or `0`, no multiprocessing is used and the operation runs in the main process.
+ - If greater than `1`, one or multiple worker processes are used to process data in parallel.
+ Note: The function passed to `map()` must be picklable for multiprocessing to work correctly
+ (i.e., prefer functions defined at the top level of a module, not inside another function or class).
+ desc (`str`, *optional*, defaults to `None`):
+ Meaningful description to be displayed alongside with the progress bar while filtering examples.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds.filter(lambda x: x["label"] == 1)
+ DatasetDict({
+ train: Dataset({
+ features: ['text', 'label'],
+ num_rows: 4265
+ })
+ validation: Dataset({
+ features: ['text', 'label'],
+ num_rows: 533
+ })
+ test: Dataset({
+ features: ['text', 'label'],
+ num_rows: 533
+ })
+ })
+ ```
+ """
+ self._check_values_type()
+ if cache_file_names is None:
+ cache_file_names = dict.fromkeys(self)
+ return DatasetDict(
+ {
+ k: dataset.filter(
+ function=function,
+ with_indices=with_indices,
+ with_rank=with_rank,
+ input_columns=input_columns,
+ batched=batched,
+ batch_size=batch_size,
+ keep_in_memory=keep_in_memory,
+ load_from_cache_file=load_from_cache_file,
+ cache_file_name=cache_file_names[k],
+ writer_batch_size=writer_batch_size,
+ fn_kwargs=fn_kwargs,
+ num_proc=num_proc,
+ desc=desc,
+ )
+ for k, dataset in self.items()
+ }
+ )
+
+ def flatten_indices(
+ self,
+ keep_in_memory: bool = False,
+ cache_file_names: Optional[dict[str, Optional[str]]] = None,
+ writer_batch_size: Optional[int] = 1000,
+ features: Optional[Features] = None,
+ disable_nullable: bool = False,
+ num_proc: Optional[int] = None,
+ new_fingerprint: Optional[str] = None,
+ ) -> "DatasetDict":
+ """Create and cache a new Dataset by flattening the indices mapping.
+
+ Args:
+ keep_in_memory (`bool`, defaults to `False`):
+ Keep the dataset in memory instead of writing it to a cache file.
+ cache_file_names (`Dict[str, str]`, *optional*, default `None`):
+ Provide the name of a path for the cache file. It is used to store the
+ results of the computation instead of the automatically generated cache file name.
+ You have to provide one `cache_file_name` per dataset in the dataset dictionary.
+ writer_batch_size (`int`, defaults to `1000`):
+ Number of rows per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`.
+ features (`Optional[datasets.Features]`, defaults to `None`):
+ Use a specific [`Features`] to store the cache file
+ instead of the automatically generated one.
+ disable_nullable (`bool`, defaults to `False`):
+ Allow null values in the table.
+ num_proc (`int`, optional, default `None`):
+ Max number of processes when generating cache. Already cached shards are loaded sequentially
+ new_fingerprint (`str`, *optional*, defaults to `None`):
+ The new fingerprint of the dataset after transform.
+ If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments
+ """
+ self._check_values_type()
+ if cache_file_names is None:
+ cache_file_names = dict.fromkeys(self)
+ return DatasetDict(
+ {
+ k: dataset.flatten_indices(
+ keep_in_memory=keep_in_memory,
+ cache_file_name=cache_file_names[k],
+ writer_batch_size=writer_batch_size,
+ features=features,
+ disable_nullable=disable_nullable,
+ num_proc=num_proc,
+ new_fingerprint=new_fingerprint,
+ )
+ for k, dataset in self.items()
+ }
+ )
+
+ def sort(
+ self,
+ column_names: Union[str, Sequence[str]],
+ reverse: Union[bool, Sequence[bool]] = False,
+ null_placement: str = "at_end",
+ keep_in_memory: bool = False,
+ load_from_cache_file: Optional[bool] = None,
+ indices_cache_file_names: Optional[dict[str, Optional[str]]] = None,
+ writer_batch_size: Optional[int] = 1000,
+ ) -> "DatasetDict":
+ """Create a new dataset sorted according to a single or multiple columns.
+
+ Args:
+ column_names (`Union[str, Sequence[str]]`):
+ Column name(s) to sort by.
+ reverse (`Union[bool, Sequence[bool]]`, defaults to `False`):
+ If `True`, sort by descending order rather than ascending. If a single bool is provided,
+ the value is applied to the sorting of all column names. Otherwise a list of bools with the
+ same length and order as column_names must be provided.
+ null_placement (`str`, defaults to `at_end`):
+ Put `None` values at the beginning if `at_start` or `first` or at the end if `at_end` or `last`
+ keep_in_memory (`bool`, defaults to `False`):
+ Keep the sorted indices in memory instead of writing it to a cache file.
+ load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled):
+ If a cache file storing the sorted indices
+ can be identified, use it instead of recomputing.
+ indices_cache_file_names (`[Dict[str, str]]`, *optional*, defaults to `None`):
+ Provide the name of a path for the cache file. It is used to store the
+ indices mapping instead of the automatically generated cache file name.
+ You have to provide one `cache_file_name` per dataset in the dataset dictionary.
+ writer_batch_size (`int`, defaults to `1000`):
+ Number of rows per write operation for the cache file writer.
+ Higher value gives smaller cache files, lower value consume less temporary memory.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset('cornell-movie-review-data/rotten_tomatoes')
+ >>> ds['train']['label'][:10]
+ [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
+ >>> sorted_ds = ds.sort('label')
+ >>> sorted_ds['train']['label'][:10]
+ [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+ >>> another_sorted_ds = ds.sort(['label', 'text'], reverse=[True, False])
+ >>> another_sorted_ds['train']['label'][:10]
+ [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
+ ```
+ """
+ self._check_values_type()
+ if indices_cache_file_names is None:
+ indices_cache_file_names = dict.fromkeys(self)
+ return DatasetDict(
+ {
+ k: dataset.sort(
+ column_names=column_names,
+ reverse=reverse,
+ null_placement=null_placement,
+ keep_in_memory=keep_in_memory,
+ load_from_cache_file=load_from_cache_file,
+ indices_cache_file_name=indices_cache_file_names[k],
+ writer_batch_size=writer_batch_size,
+ )
+ for k, dataset in self.items()
+ }
+ )
+
+ def shuffle(
+ self,
+ seeds: Optional[Union[int, dict[str, Optional[int]]]] = None,
+ seed: Optional[int] = None,
+ generators: Optional[dict[str, np.random.Generator]] = None,
+ keep_in_memory: bool = False,
+ load_from_cache_file: Optional[bool] = None,
+ indices_cache_file_names: Optional[dict[str, Optional[str]]] = None,
+ writer_batch_size: Optional[int] = 1000,
+ ) -> "DatasetDict":
+ """Create a new Dataset where the rows are shuffled.
+
+ The transformation is applied to all the datasets of the dataset dictionary.
+
+ Currently shuffling uses numpy random generators.
+ You can either supply a NumPy BitGenerator to use, or a seed to initiate NumPy's default random generator (PCG64).
+
+ Args:
+ seeds (`Dict[str, int]` or `int`, *optional*):
+ A seed to initialize the default BitGenerator if `generator=None`.
+ If `None`, then fresh, unpredictable entropy will be pulled from the OS.
+ If an `int` or `array_like[ints]` is passed, then it will be passed to SeedSequence to derive the initial BitGenerator state.
+ You can provide one `seed` per dataset in the dataset dictionary.
+ seed (`int`, *optional*):
+ A seed to initialize the default BitGenerator if `generator=None`. Alias for seeds (a `ValueError` is raised if both are provided).
+ generators (`Dict[str, *optional*, np.random.Generator]`):
+ Numpy random Generator to use to compute the permutation of the dataset rows.
+ If `generator=None` (default), uses `np.random.default_rng` (the default BitGenerator (PCG64) of NumPy).
+ You have to provide one `generator` per dataset in the dataset dictionary.
+ keep_in_memory (`bool`, defaults to `False`):
+ Keep the dataset in memory instead of writing it to a cache file.
+ load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled):
+ If a cache file storing the current computation from `function`
+ can be identified, use it instead of recomputing.
+ indices_cache_file_names (`Dict[str, str]`, *optional*):
+ Provide the name of a path for the cache file. It is used to store the
+ indices mappings instead of the automatically generated cache file name.
+ You have to provide one `cache_file_name` per dataset in the dataset dictionary.
+ writer_batch_size (`int`, defaults to `1000`):
+ Number of rows per write operation for the cache file writer.
+ This value is a good trade-off between memory usage during the processing, and processing speed.
+ Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds["train"]["label"][:10]
+ [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
+
+ # set a seed
+ >>> shuffled_ds = ds.shuffle(seed=42)
+ >>> shuffled_ds["train"]["label"][:10]
+ [0, 1, 0, 1, 0, 0, 0, 0, 0, 0]
+ ```
+ """
+ self._check_values_type()
+ if seed is not None and seeds is not None:
+ raise ValueError("Please specify seed or seeds, but not both")
+ seeds = seed if seed is not None else seeds
+ if seeds is None:
+ seeds = dict.fromkeys(self)
+ elif not isinstance(seeds, dict):
+ seeds = dict.fromkeys(self, seeds)
+ if generators is None:
+ generators = dict.fromkeys(self)
+ if indices_cache_file_names is None:
+ indices_cache_file_names = dict.fromkeys(self)
+ return DatasetDict(
+ {
+ k: dataset.shuffle(
+ seed=seeds[k],
+ generator=generators[k],
+ keep_in_memory=keep_in_memory,
+ load_from_cache_file=load_from_cache_file,
+ indices_cache_file_name=indices_cache_file_names[k],
+ writer_batch_size=writer_batch_size,
+ )
+ for k, dataset in self.items()
+ }
+ )
+
+ def save_to_disk(
+ self,
+ dataset_dict_path: PathLike,
+ max_shard_size: Optional[Union[str, int]] = None,
+ num_shards: Optional[dict[str, int]] = None,
+ num_proc: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ ):
+ """
+ Saves a dataset dict to a filesystem using `fsspec.spec.AbstractFileSystem`.
+
+ For [`Image`], [`Audio`] and [`Video`] data:
+
+ All the Image(), Audio() and Video() data are stored in the arrow files.
+ If you want to store paths or urls, please use the Value("string") type.
+
+ Args:
+ dataset_dict_path (`path-like`):
+ Path (e.g. `dataset/train`) or remote URI (e.g. `s3://my-bucket/dataset/train`)
+ of the dataset dict directory where the dataset dict will be saved to.
+ max_shard_size (`int` or `str`, *optional*, defaults to `"500MB"`):
+ The maximum size of the dataset shards to be saved to the filesystem. If expressed as a string, needs to be digits followed by a unit
+ (like `"50MB"`).
+ num_shards (`Dict[str, int]`, *optional*):
+ Number of shards to write. By default the number of shards depends on `max_shard_size` and `num_proc`.
+ You need to provide the number of shards for each dataset in the dataset dictionary.
+ Use a dictionary to define a different num_shards for each split.
+
+
+ num_proc (`int`, *optional*, default `None`):
+ Number of processes when downloading and generating the dataset locally.
+ Multiprocessing is disabled by default.
+
+
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+
+
+
+ Example:
+
+ ```python
+ >>> dataset_dict.save_to_disk("path/to/dataset/directory")
+ >>> dataset_dict.save_to_disk("path/to/dataset/directory", max_shard_size="1GB")
+ >>> dataset_dict.save_to_disk("path/to/dataset/directory", num_shards={"train": 1024, "test": 8})
+ ```
+ """
+ fs: fsspec.AbstractFileSystem
+ fs, _ = url_to_fs(dataset_dict_path, **(storage_options or {}))
+
+ if num_shards is None:
+ num_shards = dict.fromkeys(self)
+ elif not isinstance(num_shards, dict):
+ raise ValueError(
+ "Please provide one `num_shards` per dataset in the dataset dictionary, e.g. {{'train': 128, 'test': 4}}"
+ )
+
+ fs.makedirs(dataset_dict_path, exist_ok=True)
+
+ with fs.open(
+ posixpath.join(dataset_dict_path, config.DATASETDICT_JSON_FILENAME),
+ "w",
+ encoding="utf-8",
+ ) as f:
+ json.dump({"splits": list(self)}, f)
+ for k, dataset in self.items():
+ dataset.save_to_disk(
+ posixpath.join(dataset_dict_path, k),
+ num_shards=num_shards.get(k),
+ max_shard_size=max_shard_size,
+ num_proc=num_proc,
+ storage_options=storage_options,
+ )
+
+ @staticmethod
+ def load_from_disk(
+ dataset_dict_path: PathLike,
+ keep_in_memory: Optional[bool] = None,
+ storage_options: Optional[dict] = None,
+ ) -> "DatasetDict":
+ """
+ Load a dataset that was previously saved using [`save_to_disk`] from a filesystem using `fsspec.spec.AbstractFileSystem`.
+
+ Args:
+ dataset_dict_path (`path-like`):
+ Path (e.g. `"dataset/train"`) or remote URI (e.g. `"s3//my-bucket/dataset/train"`)
+ of the dataset dict directory where the dataset dict will be loaded from.
+ keep_in_memory (`bool`, defaults to `None`):
+ Whether to copy the dataset in-memory. If `None`, the
+ dataset will not be copied in-memory unless explicitly enabled by setting
+ `datasets.config.IN_MEMORY_MAX_SIZE` to nonzero. See more details in the
+ [improve performance](../cache#improve-performance) section.
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+
+
+
+ Returns:
+ [`DatasetDict`]
+
+ Example:
+
+ ```py
+ >>> ds = load_from_disk('path/to/dataset/directory')
+ ```
+ """
+ fs: fsspec.AbstractFileSystem
+ fs, dataset_dict_path = url_to_fs(dataset_dict_path, **(storage_options or {}))
+
+ dataset_dict_json_path = posixpath.join(dataset_dict_path, config.DATASETDICT_JSON_FILENAME)
+ dataset_state_json_path = posixpath.join(dataset_dict_path, config.DATASET_STATE_JSON_FILENAME)
+ dataset_info_path = posixpath.join(dataset_dict_path, config.DATASET_INFO_FILENAME)
+ if not fs.isfile(dataset_dict_json_path):
+ if fs.isfile(dataset_info_path) and fs.isfile(dataset_state_json_path):
+ raise FileNotFoundError(
+ f"No such file: '{dataset_dict_json_path}'. Expected to load a `DatasetDict` object, but got a `Dataset`. Please use either `datasets.load_from_disk` or `Dataset.load_from_disk` instead."
+ )
+ raise FileNotFoundError(
+ f"No such file: '{dataset_dict_json_path}'. Expected to load a `DatasetDict` object, but provided path is not a `DatasetDict`."
+ )
+
+ with fs.open(dataset_dict_json_path, "r", encoding="utf-8") as f:
+ splits = json.load(f)["splits"]
+
+ dataset_dict = DatasetDict()
+ for k in splits:
+ dataset_dict_split_path = posixpath.join(fs.unstrip_protocol(dataset_dict_path), k)
+ dataset_dict[k] = Dataset.load_from_disk(
+ dataset_dict_split_path,
+ keep_in_memory=keep_in_memory,
+ storage_options=storage_options,
+ )
+ return dataset_dict
+
+ @staticmethod
+ def from_csv(
+ path_or_paths: dict[str, PathLike],
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ **kwargs,
+ ) -> "DatasetDict":
+ """Create [`DatasetDict`] from CSV file(s).
+
+ Args:
+ path_or_paths (`dict` of path-like):
+ Path(s) of the CSV file(s).
+ features ([`Features`], *optional*):
+ Dataset features.
+ cache_dir (str, *optional*, defaults to `"~/.cache/huggingface/datasets"`):
+ Directory to cache data.
+ keep_in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+ **kwargs (additional keyword arguments):
+ Keyword arguments to be passed to [`pandas.read_csv`].
+
+ Returns:
+ [`DatasetDict`]
+
+ Example:
+
+ ```py
+ >>> from datasets import DatasetDict
+ >>> ds = DatasetDict.from_csv({'train': 'path/to/dataset.csv'})
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.csv import CsvDatasetReader
+
+ return CsvDatasetReader(
+ path_or_paths,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ **kwargs,
+ ).read()
+
+ @staticmethod
+ def from_json(
+ path_or_paths: dict[str, PathLike],
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ **kwargs,
+ ) -> "DatasetDict":
+ """Create [`DatasetDict`] from JSON Lines file(s).
+
+ Args:
+ path_or_paths (`path-like` or list of `path-like`):
+ Path(s) of the JSON Lines file(s).
+ features ([`Features`], *optional*):
+ Dataset features.
+ cache_dir (str, *optional*, defaults to `"~/.cache/huggingface/datasets"`):
+ Directory to cache data.
+ keep_in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+ **kwargs (additional keyword arguments):
+ Keyword arguments to be passed to [`JsonConfig`].
+
+ Returns:
+ [`DatasetDict`]
+
+ Example:
+
+ ```py
+ >>> from datasets import DatasetDict
+ >>> ds = DatasetDict.from_json({'train': 'path/to/dataset.json'})
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.json import JsonDatasetReader
+
+ return JsonDatasetReader(
+ path_or_paths,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ **kwargs,
+ ).read()
+
+ @staticmethod
+ def from_parquet(
+ path_or_paths: dict[str, PathLike],
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ columns: Optional[list[str]] = None,
+ **kwargs,
+ ) -> "DatasetDict":
+ """Create [`DatasetDict`] from Parquet file(s).
+
+ Args:
+ path_or_paths (`dict` of path-like):
+ Path(s) of the CSV file(s).
+ features ([`Features`], *optional*):
+ Dataset features.
+ cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`):
+ Directory to cache data.
+ keep_in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+ columns (`list[str]`, *optional*):
+ If not `None`, only these columns will be read from the file.
+ A column name may be a prefix of a nested field, e.g. 'a' will select
+ 'a.b', 'a.c', and 'a.d.e'.
+ **kwargs (additional keyword arguments):
+ Keyword arguments to be passed to [`ParquetConfig`].
+
+ Returns:
+ [`DatasetDict`]
+
+ Example:
+
+ ```py
+ >>> from datasets import DatasetDict
+ >>> ds = DatasetDict.from_parquet({'train': 'path/to/dataset/parquet'})
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.parquet import ParquetDatasetReader
+
+ return ParquetDatasetReader(
+ path_or_paths,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ columns=columns,
+ **kwargs,
+ ).read()
+
+ @staticmethod
+ def from_text(
+ path_or_paths: dict[str, PathLike],
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ **kwargs,
+ ) -> "DatasetDict":
+ """Create [`DatasetDict`] from text file(s).
+
+ Args:
+ path_or_paths (`dict` of path-like):
+ Path(s) of the text file(s).
+ features ([`Features`], *optional*):
+ Dataset features.
+ cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`):
+ Directory to cache data.
+ keep_in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+ **kwargs (additional keyword arguments):
+ Keyword arguments to be passed to [`TextConfig`].
+
+ Returns:
+ [`DatasetDict`]
+
+ Example:
+
+ ```py
+ >>> from datasets import DatasetDict
+ >>> ds = DatasetDict.from_text({'train': 'path/to/dataset.txt'})
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.text import TextDatasetReader
+
+ return TextDatasetReader(
+ path_or_paths,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ **kwargs,
+ ).read()
+
+ @is_documented_by(Dataset.align_labels_with_mapping)
+ def align_labels_with_mapping(self, label2id: dict, label_column: str) -> "DatasetDict":
+ self._check_values_type()
+ return DatasetDict(
+ {
+ k: dataset.align_labels_with_mapping(label2id=label2id, label_column=label_column)
+ for k, dataset in self.items()
+ }
+ )
+
+ def push_to_hub(
+ self,
+ repo_id: str,
+ config_name: str = "default",
+ set_default: Optional[bool] = None,
+ data_dir: Optional[str] = None,
+ commit_message: Optional[str] = None,
+ commit_description: Optional[str] = None,
+ private: Optional[bool] = None,
+ token: Optional[str] = None,
+ revision: Optional[str] = None,
+ create_pr: Optional[bool] = False,
+ max_shard_size: Optional[Union[int, str]] = None,
+ num_shards: Optional[dict[str, int]] = None,
+ embed_external_files: bool = True,
+ num_proc: Optional[int] = None,
+ ) -> Optional[CommitInfo]:
+ """Pushes the [`DatasetDict`] to the hub as a Parquet dataset.
+ The [`DatasetDict`] is pushed using HTTP requests and does not need to have neither git or git-lfs installed.
+
+ Each dataset split will be pushed independently. The pushed dataset will keep the original split names.
+
+ The resulting Parquet files are self-contained by default: if your dataset contains [`Image`] or [`Audio`]
+ data, the Parquet files will store the bytes of your images or audio files.
+ You can disable this by setting `embed_external_files` to False.
+
+ Args:
+ repo_id (`str`):
+ The ID of the repository to push to in the following format: `/` or
+ `/`. Also accepts ``, which will default to the namespace
+ of the logged-in user.
+
+ It could also be a location inside a bucket, e.g. `buckets///...`
+ config_name (`str`):
+ Configuration name of a dataset. Defaults to "default".
+ set_default (`bool`, *optional*):
+ Whether to set this configuration as the default one. Otherwise, the default configuration is the one
+ named "default".
+ data_dir (`str`, *optional*):
+ Directory name that will contain the uploaded data files. Defaults to the `config_name` if different
+ from "default", else "data".
+
+
+ commit_message (`str`, *optional*):
+ Message to commit while pushing. Will default to `"Upload dataset"`.
+ commit_description (`str`, *optional*):
+ Description of the commit that will be created.
+ Additionally, description of the PR if a PR is created (`create_pr` is True).
+
+
+ private (`bool`, *optional*):
+ Whether to make the repo private. If `None` (default), the repo will be public unless the
+ organization's default is private. This value is ignored if the repo already exists.
+ token (`str`, *optional*):
+ An optional authentication token for the Hugging Face Hub. If no token is passed, will default
+ to the token saved locally when logging in with `huggingface-cli login`. Will raise an error
+ if no token is passed and the user is not logged-in.
+ revision (`str`, *optional*):
+ Branch to push the uploaded files to. Defaults to the `"main"` branch.
+
+
+ create_pr (`bool`, *optional*, defaults to `False`):
+ Whether to create a PR with the uploaded files or directly commit.
+
+
+ max_shard_size (`int` or `str`, *optional*, defaults to `"500MB"`):
+ The maximum size of the dataset shards to be uploaded to the hub. If expressed as a string, needs to be digits followed by a unit
+ (like `"500MB"` or `"1GB"`).
+ num_shards (`Dict[str, int]`, *optional*):
+ Number of shards to write. By default, the number of shards depends on `max_shard_size`.
+ Use a dictionary to define a different num_shards for each split.
+
+
+ embed_external_files (`bool`, defaults to `True`):
+ Whether to embed file bytes in the shards.
+ In particular, this will do the following before the push for the fields of type:
+
+ - [`Audio`] and [`Image`] removes local path information and embed file content in the Parquet files.
+ num_proc (`int`, *optional*, defaults to `None`):
+ Number of processes when preparing and uploading the dataset.
+ This is helpful if the dataset is made of many samples or media files to embed.
+ I uses "spawn" context to work with hf_xet, the rust client for fast uploads to HF.
+ Multiprocessing is disabled by default.
+
+
+
+ Return:
+ huggingface_hub.CommitInfo
+
+ Example:
+
+ ```python
+ >>> dataset_dict.push_to_hub("/")
+ >>> dataset_dict.push_to_hub("/", private=True)
+ >>> dataset_dict.push_to_hub("/", max_shard_size="1GB")
+ >>> dataset_dict.push_to_hub("/", num_shards={"train": 1024, "test": 8})
+ ```
+
+ If you want to add a new configuration (or subset) to a dataset (e.g. if the dataset has multiple tasks/versions/languages):
+
+ ```python
+ >>> english_dataset.push_to_hub("/", "en")
+ >>> french_dataset.push_to_hub("/", "fr")
+ >>> # later
+ >>> english_dataset = load_dataset("/", "en")
+ >>> french_dataset = load_dataset("/", "fr")
+ ```
+ """
+ if num_shards is None:
+ num_shards = dict.fromkeys(self)
+ elif not isinstance(num_shards, dict):
+ raise ValueError(
+ "Please provide one `num_shards` per dataset in the dataset dictionary, e.g. {{'train': 128, 'test': 4}}"
+ )
+
+ self._check_values_type()
+ self._check_values_features()
+ if config_name == "data":
+ raise ValueError("`config_name` cannot be 'data'. Please, choose another name for configuration.")
+
+ if max_shard_size is not None and num_shards is not None and any(x is not None for x in num_shards.values()):
+ raise ValueError(
+ "Failed to push_to_hub: please specify either max_shard_size or num_shards, but not both."
+ )
+
+ for split in self:
+ if not re.match(_split_re, split):
+ raise ValueError(f"Split name should match '{_split_re}' but got '{split}'.")
+
+ if not data_dir:
+ data_dir = config_name if config_name != "default" else "data" # for backward compatibility
+
+ api = HfApi(endpoint=config.HF_ENDPOINT, token=token, library_name="datasets", library_version=__version__)
+ if repo_id.startswith("buckets/"):
+ if BucketNotFoundError is None:
+ raise ImportError("Pushing datasets to buckets requires huggingface_hub>=1.6.0")
+ _, _namespace, _bucket_name, *_path_segments = repo_id.split("/")
+ try:
+ bucket_id = api.bucket_info(_namespace + "/" + _bucket_name).id
+ except BucketNotFoundError:
+ bucket_url = api.create_bucket(_namespace + "/" + _bucket_name, private=private, exist_ok=True)
+ bucket_id = bucket_url.bucket_id
+ path = "/".join(s for s in _path_segments if s)
+ return _push_to_bucket(
+ self,
+ bucket_id=bucket_id,
+ path=path,
+ config_name=config_name,
+ set_default=set_default,
+ data_dir=data_dir,
+ token=token,
+ max_shard_size=max_shard_size,
+ num_shards=num_shards,
+ embed_external_files=embed_external_files,
+ num_proc=num_proc,
+ )
+ else:
+ try:
+ repo_id = api.repo_info(repo_id, repo_type="dataset").id
+ except RepositoryNotFoundError:
+ repo_url = api.create_repo(
+ repo_id,
+ repo_type="dataset",
+ private=private,
+ exist_ok=True,
+ )
+ repo_id = repo_url.repo_id
+
+ if revision is not None and not revision.startswith("refs/pr/"):
+ # We do not call create_branch for a PR reference: 400 Bad Request
+ api.create_branch(repo_id, branch=revision, repo_type="dataset", exist_ok=True)
+ return _push_to_repo(
+ self,
+ repo_id=repo_id,
+ config_name=config_name,
+ set_default=set_default,
+ data_dir=data_dir,
+ commit_message=commit_message,
+ commit_description=commit_description,
+ token=token,
+ revision=revision,
+ create_pr=create_pr,
+ max_shard_size=max_shard_size,
+ num_shards=num_shards,
+ embed_external_files=embed_external_files,
+ num_proc=num_proc,
+ )
+
+
+class IterableDatasetDict(dict[Union[str, NamedSplit], IterableDataset]):
+ def _check_values_type(self):
+ for dataset in self.values():
+ if not isinstance(dataset, IterableDataset):
+ raise TypeError(f"Values in `DatasetDict` should be of type `Dataset` but got type '{type(dataset)}'")
+
+ def _check_values_features(self):
+ items = [(key, dataset._resolve_features()) for key, dataset in self.items()]
+ for item_a, item_b in zip(items[:-1], items[1:]):
+ if item_a[1].features != item_b[1].features:
+ raise ValueError(
+ f"All datasets in `DatasetDict` should have the same features but features for '{item_a[0]}' and '{item_b[0]}' don't match: {item_a[1].features} != {item_b[1].features}"
+ )
+
+ def __repr__(self):
+ repr = "\n".join([f"{k}: {v}" for k, v in self.items()])
+ repr = re.sub(r"^", " " * 4, repr, count=0, flags=re.M)
+ return f"IterableDatasetDict({{\n{repr}\n}})"
+
+ @property
+ def num_columns(self) -> dict[str, Optional[int]]:
+ """Number of columns in each split of the dataset.
+ This can contain None valies if some splits have unknown features (e.g. after a map() operation).
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds.num_columns
+ {'test': 2, 'train': 2, 'validation': 2}
+ ```
+ """
+ self._check_values_type()
+ return {k: dataset.num_columns for k, dataset in self.items()}
+
+ @property
+ def column_names(self) -> dict[str, Optional[list[str]]]:
+ """Names of the columns in each split of the dataset.
+ This can contain None valies if some splits have unknown features (e.g. after a map() operation).
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+ >>> ds.column_names
+ {'test': ['text', 'label'],
+ 'train': ['text', 'label'],
+ 'validation': ['text', 'label']}
+ ```
+ """
+ self._check_values_type()
+ return {k: dataset.column_names for k, dataset in self.items()}
+
+ def with_format(
+ self,
+ type: Optional[str] = None,
+ ) -> "IterableDatasetDict":
+ """
+ Return a dataset with the specified format.
+
+ Args:
+
+ type (`str`, *optional*):
+ Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`.
+ `None` means it returns python objects (default).
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> from transformers import AutoTokenizer
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation", streaming=True)
+ >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+ >>> ds = ds.map(lambda x: tokenizer(x['text'], truncation=True, padding=True), batched=True)
+ >>> ds = ds.with_format("torch")
+ >>> next(iter(ds))
+ {'text': 'compassionately explores the seemingly irreconcilable situation between conservative christian parents and their estranged gay and lesbian children .',
+ 'label': tensor(1),
+ 'input_ids': tensor([ 101, 18027, 16310, 16001, 1103, 9321, 178, 11604, 7235, 6617,
+ 1742, 2165, 2820, 1206, 6588, 22572, 12937, 1811, 2153, 1105,
+ 1147, 12890, 19587, 6463, 1105, 15026, 1482, 119, 102, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0]),
+ 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
+ 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])}
+ ```
+ """
+ return IterableDatasetDict({k: dataset.with_format(type=type) for k, dataset in self.items()})
+
+ def map(
+ self,
+ function: Optional[Callable] = None,
+ with_indices: bool = False,
+ with_split: bool = False,
+ input_columns: Optional[Union[str, list[str]]] = None,
+ batched: bool = False,
+ batch_size: int = 1000,
+ drop_last_batch: bool = False,
+ remove_columns: Optional[Union[str, list[str]]] = None,
+ fn_kwargs: Optional[dict] = None,
+ ) -> "IterableDatasetDict":
+ """
+ Apply a function to all the examples in the iterable dataset (individually or in batches) and update them.
+ If your function returns a column that already exists, then it overwrites it.
+ The function is applied on-the-fly on the examples when iterating over the dataset.
+ The transformation is applied to all the datasets of the dataset dictionary.
+
+ You can specify whether the function should be batched or not with the `batched` parameter:
+
+ - If batched is `False`, then the function takes 1 example in and should return 1 example.
+ An example is a dictionary, e.g. `{"text": "Hello there !"}`.
+ - If batched is `True` and `batch_size` is 1, then the function takes a batch of 1 example as input and can return a batch with 1 or more examples.
+ A batch is a dictionary, e.g. a batch of 1 example is `{"text": ["Hello there !"]}`.
+ - If batched is `True` and `batch_size` is `n` > 1, then the function takes a batch of `n` examples as input and can return a batch with `n` examples, or with an arbitrary number of examples.
+ Note that the last batch may have less than `n` examples.
+ A batch is a dictionary, e.g. a batch of `n` examples is `{"text": ["Hello there !"] * n}`.
+
+ If the function is asynchronous, then `map` will run your function in parallel, with up to one thousand simultaneous calls.
+ It is recommended to use a `asyncio.Semaphore` in your function if you want to set a maximum number of operations that can run at the same time.
+
+ Args:
+ function (`Callable`, *optional*, defaults to `None`):
+ Function applied on-the-fly on the examples when you iterate on the dataset.
+ It must have one of the following signatures:
+
+ - `function(example: Dict[str, Any]) -> Dict[str, Any]` if `batched=False` and `with_indices=False`
+ - `function(example: Dict[str, Any], idx: int) -> Dict[str, Any]` if `batched=False` and `with_indices=True`
+ - `function(batch: Dict[str, list]) -> Dict[str, list]` if `batched=True` and `with_indices=False`
+ - `function(batch: Dict[str, list], indices: list[int]) -> Dict[str, list]` if `batched=True` and `with_indices=True`
+
+ For advanced usage, the function can also return a `pyarrow.Table`.
+ If the function is asynchronous, then `map` will run your function in parallel.
+ Moreover if your function returns nothing (`None`), then `map` will run your function and return the dataset unchanged.
+ If no function is provided, default to identity function: `lambda x: x`.
+ with_indices (`bool`, defaults to `False`):
+ Provide example indices to `function`. Note that in this case the signature of `function` should be `def function(example, idx[, rank]): ...`.
+ input_columns (`[Union[str, list[str]]]`, *optional*, defaults to `None`):
+ The columns to be passed into `function`
+ as positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument.
+ batched (`bool`, defaults to `False`):
+ Provide batch of examples to `function`.
+ batch_size (`int`, *optional*, defaults to `1000`):
+ Number of examples per batch provided to `function` if `batched=True`.
+ drop_last_batch (`bool`, defaults to `False`):
+ Whether a last batch smaller than the `batch_size` should be
+ dropped instead of being processed by the function.
+ remove_columns (`[list[str]]`, *optional*, defaults to `None`):
+ Remove a selection of columns while doing the mapping.
+ Columns will be removed before updating the examples with the output of `function`, i.e. if `function` is adding
+ columns with names in `remove_columns`, these columns will be kept.
+ fn_kwargs (`Dict`, *optional*, defaults to `None`):
+ Keyword arguments to be passed to `function`
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True)
+ >>> def add_prefix(example):
+ ... example["text"] = "Review: " + example["text"]
+ ... return example
+ >>> ds = ds.map(add_prefix)
+ >>> next(iter(ds["train"]))
+ {'label': 1,
+ 'text': 'Review: the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}
+ ```
+ """
+
+ dataset_dict = {}
+ for split, dataset in self.items():
+ if with_split:
+ function = bind(function, split)
+
+ dataset_dict[split] = dataset.map(
+ function=function,
+ with_indices=with_indices,
+ input_columns=input_columns,
+ batched=batched,
+ batch_size=batch_size,
+ drop_last_batch=drop_last_batch,
+ remove_columns=remove_columns,
+ fn_kwargs=fn_kwargs,
+ )
+
+ if with_split:
+ function = function.func
+
+ return IterableDatasetDict(dataset_dict)
+
+ def filter(
+ self,
+ function: Optional[Callable] = None,
+ with_indices=False,
+ input_columns: Optional[Union[str, list[str]]] = None,
+ batched: bool = False,
+ batch_size: Optional[int] = 1000,
+ fn_kwargs: Optional[dict] = None,
+ ) -> "IterableDatasetDict":
+ """Apply a filter function to all the elements so that the dataset only includes examples according to the filter function.
+ The filtering is done on-the-fly when iterating over the dataset.
+ The filtering is applied to all the datasets of the dataset dictionary.
+
+ Args:
+ function (`Callable`):
+ Callable with one of the following signatures:
+
+ - `function(example: Dict[str, Any]) -> bool` if `with_indices=False, batched=False`
+ - `function(example: Dict[str, Any], indices: int) -> bool` if `with_indices=True, batched=False`
+ - `function(example: Dict[str, list]) -> list[bool]` if `with_indices=False, batched=True`
+ - `function(example: Dict[str, list], indices: list[int]) -> list[bool]` if `with_indices=True, batched=True`
+
+ If no function is provided, defaults to an always True function: `lambda x: True`.
+ with_indices (`bool`, defaults to `False`):
+ Provide example indices to `function`. Note that in this case the signature of `function` should be `def function(example, idx): ...`.
+ input_columns (`str` or `list[str]`, *optional*):
+ The columns to be passed into `function` as
+ positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument.
+ batched (`bool`, defaults to `False`):
+ Provide batch of examples to `function`
+ batch_size (`int`, *optional*, defaults to `1000`):
+ Number of examples per batch provided to `function` if `batched=True`.
+ fn_kwargs (`Dict`, *optional*, defaults to `None`):
+ Keyword arguments to be passed to `function`
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True)
+ >>> ds = ds.filter(lambda x: x["label"] == 0)
+ >>> list(ds["train"].take(3))
+ [{'label': 0, 'text': 'Review: simplistic , silly and tedious .'},
+ {'label': 0,
+ 'text': "Review: it's so laddish and juvenile , only teenage boys could possibly find it funny ."},
+ {'label': 0,
+ 'text': 'Review: exploitative and largely devoid of the depth or sophistication that would make watching such a graphic treatment of the crimes bearable .'}]
+ ```
+ """
+ return IterableDatasetDict(
+ {
+ k: dataset.filter(
+ function=function,
+ with_indices=with_indices,
+ input_columns=input_columns,
+ batched=batched,
+ batch_size=batch_size,
+ fn_kwargs=fn_kwargs,
+ )
+ for k, dataset in self.items()
+ }
+ )
+
+ def shuffle(
+ self,
+ seed=None,
+ generator: Optional[np.random.Generator] = None,
+ buffer_size: int = 1000,
+ ) -> "IterableDatasetDict":
+ """
+ Randomly shuffles the elements of this dataset.
+ The shuffling is applied to all the datasets of the dataset dictionary.
+
+ This dataset fills a buffer with buffer_size elements, then randomly samples elements from this buffer,
+ replacing the selected elements with new elements. For perfect shuffling, a buffer size greater than or
+ equal to the full size of the dataset is required.
+
+ For instance, if your dataset contains 10,000 elements but `buffer_size` is set to 1000, then `shuffle` will
+ initially select a random element from only the first 1000 elements in the buffer. Once an element is
+ selected, its space in the buffer is replaced by the next (i.e. 1,001-st) element,
+ maintaining the 1000 element buffer.
+
+ If the dataset is made of several shards, it also does `shuffle` the order of the shards.
+ However if the order has been fixed by using [`~datasets.IterableDataset.skip`] or [`~datasets.IterableDataset.take`]
+ then the order of the shards is kept unchanged.
+
+ Args:
+ seed (`int`, *optional*, defaults to `None`):
+ Random seed that will be used to shuffle the dataset.
+ It is used to sample from the shuffle buffer and also to shuffle the data shards.
+ generator (`numpy.random.Generator`, *optional*):
+ Numpy random Generator to use to compute the permutation of the dataset rows.
+ If `generator=None` (default), uses `np.random.default_rng` (the default BitGenerator (PCG64) of NumPy).
+ buffer_size (`int`, defaults to `1000`):
+ Size of the buffer.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True)
+ >>> list(ds["train"].take(3))
+ [{'label': 1,
+ 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'},
+ {'label': 1,
+ 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'},
+ {'label': 1, 'text': 'effective but too-tepid biopic'}]
+ >>> ds = ds.shuffle(seed=42)
+ >>> list(ds["train"].take(3))
+ [{'label': 1,
+ 'text': "a sports movie with action that's exciting on the field and a story you care about off it ."},
+ {'label': 1,
+ 'text': 'at its best , the good girl is a refreshingly adult take on adultery . . .'},
+ {'label': 1,
+ 'text': "sam jones became a very lucky filmmaker the day wilco got dropped from their record label , proving that one man's ruin may be another's fortune ."}]
+ ```
+ """
+ return IterableDatasetDict(
+ {
+ k: dataset.shuffle(seed=seed, generator=generator, buffer_size=buffer_size)
+ for k, dataset in self.items()
+ }
+ )
+
+ def rename_column(self, original_column_name: str, new_column_name: str) -> "IterableDatasetDict":
+ """
+ Rename a column in the dataset, and move the features associated to the original column under the new column
+ name.
+ The renaming is applied to all the datasets of the dataset dictionary.
+
+ Args:
+ original_column_name (`str`):
+ Name of the column to rename.
+ new_column_name (`str`):
+ New name for the column.
+
+ Returns:
+ [`IterableDatasetDict`]: A copy of the dataset with a renamed column.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True)
+ >>> ds = ds.rename_column("text", "movie_review")
+ >>> next(iter(ds["train"]))
+ {'label': 1,
+ 'movie_review': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}
+ ```
+ """
+ return IterableDatasetDict(
+ {
+ k: dataset.rename_column(
+ original_column_name=original_column_name,
+ new_column_name=new_column_name,
+ )
+ for k, dataset in self.items()
+ }
+ )
+
+ def rename_columns(self, column_mapping: dict[str, str]) -> "IterableDatasetDict":
+ """
+ Rename several columns in the dataset, and move the features associated to the original columns under
+ the new column names.
+ The renaming is applied to all the datasets of the dataset dictionary.
+
+ Args:
+ column_mapping (`Dict[str, str]`):
+ A mapping of columns to rename to their new names.
+
+ Returns:
+ [`IterableDatasetDict`]: A copy of the dataset with renamed columns
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True)
+ >>> ds = ds.rename_columns({"text": "movie_review", "label": "rating"})
+ >>> next(iter(ds["train"]))
+ {'movie_review': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .',
+ 'rating': 1}
+ ```
+ """
+ return IterableDatasetDict(
+ {k: dataset.rename_columns(column_mapping=column_mapping) for k, dataset in self.items()}
+ )
+
+ def remove_columns(self, column_names: Union[str, list[str]]) -> "IterableDatasetDict":
+ """
+ Remove one or several column(s) in the dataset and the features associated to them.
+ The removal is done on-the-fly on the examples when iterating over the dataset.
+ The removal is applied to all the datasets of the dataset dictionary.
+
+
+ Args:
+ column_names (`Union[str, list[str]]`):
+ Name of the column(s) to remove.
+
+ Returns:
+ [`IterableDatasetDict`]: A copy of the dataset object without the columns to remove.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True)
+ >>> ds = ds.remove_columns("label")
+ >>> next(iter(ds["train"]))
+ {'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}
+ ```
+ """
+ return IterableDatasetDict({k: dataset.remove_columns(column_names) for k, dataset in self.items()})
+
+ def select_columns(self, column_names: Union[str, list[str]]) -> "IterableDatasetDict":
+ """Select one or several column(s) in the dataset and the features
+ associated to them. The selection is done on-the-fly on the examples
+ when iterating over the dataset. The selection is applied to all the
+ datasets of the dataset dictionary.
+
+
+ Args:
+ column_names (`Union[str, list[str]]`):
+ Name of the column(s) to keep.
+
+ Returns:
+ [`IterableDatasetDict`]: A copy of the dataset object with only selected columns.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True)
+ >>> ds = ds.select("text")
+ >>> next(iter(ds["train"]))
+ {'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}
+ ```
+ """
+ return IterableDatasetDict({k: dataset.select_columns(column_names) for k, dataset in self.items()})
+
+ def cast_column(self, column: str, feature: FeatureType) -> "IterableDatasetDict":
+ """Cast column to feature for decoding.
+ The type casting is applied to all the datasets of the dataset dictionary.
+
+ Args:
+ column (`str`):
+ Column name.
+ feature ([`Feature`]):
+ Target feature.
+
+ Returns:
+ [`IterableDatasetDict`]
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset, ClassLabel
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True)
+ >>> ds["train"].features
+ {'label': ClassLabel(names=['neg', 'pos']),
+ 'text': Value('string')}
+ >>> ds = ds.cast_column('label', ClassLabel(names=['bad', 'good']))
+ >>> ds["train"].features
+ {'label': ClassLabel(names=['bad', 'good']),
+ 'text': Value('string')}
+ ```
+ """
+ return IterableDatasetDict(
+ {k: dataset.cast_column(column=column, feature=feature) for k, dataset in self.items()}
+ )
+
+ def cast(
+ self,
+ features: Features,
+ ) -> "IterableDatasetDict":
+ """
+ Cast the dataset to a new set of features.
+ The type casting is applied to all the datasets of the dataset dictionary.
+
+ Args:
+ features (`Features`):
+ New features to cast the dataset to.
+ The name of the fields in the features must match the current column names.
+ The type of the data must also be convertible from one type to the other.
+ For non-trivial conversion, e.g. `string` <-> `ClassLabel` you should use [`map`] to update the Dataset.
+
+ Returns:
+ [`IterableDatasetDict`]: A copy of the dataset with casted features.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True)
+ >>> ds["train"].features
+ {'label': ClassLabel(names=['neg', 'pos']),
+ 'text': Value('string')}
+ >>> new_features = ds["train"].features.copy()
+ >>> new_features['label'] = ClassLabel(names=['bad', 'good'])
+ >>> new_features['text'] = Value('large_string')
+ >>> ds = ds.cast(new_features)
+ >>> ds["train"].features
+ {'label': ClassLabel(names=['bad', 'good']),
+ 'text': Value('large_string')}
+ ```
+ """
+ return IterableDatasetDict({k: dataset.cast(features=features) for k, dataset in self.items()})
+
+ def push_to_hub(
+ self,
+ repo_id: str,
+ config_name: str = "default",
+ set_default: Optional[bool] = None,
+ data_dir: Optional[str] = None,
+ commit_message: Optional[str] = None,
+ commit_description: Optional[str] = None,
+ private: Optional[bool] = None,
+ token: Optional[str] = None,
+ revision: Optional[str] = None,
+ create_pr: Optional[bool] = False,
+ max_shard_size: Optional[Union[int, str]] = None,
+ num_shards: Optional[dict[str, int]] = None,
+ embed_external_files: bool = True,
+ num_proc: Optional[int] = None,
+ ) -> CommitInfo:
+ """Pushes the [`IterableDatasetDict`] to the hub as a Parquet dataset.
+ The [`IterableDatasetDict`] is pushed using HTTP requests and does not need to have neither git or git-lfs installed.
+
+ Each dataset split will be pushed independently. The pushed dataset will keep the original split names.
+
+ The resulting Parquet files are self-contained by default: if your dataset contains [`Image`] or [`Audio`]
+ data, the Parquet files will store the bytes of your images or audio files.
+ You can disable this by setting `embed_external_files` to False.
+
+ Args:
+ repo_id (`str`):
+ The ID of the repository to push to in the following format: `/` or
+ `/`. Also accepts ``, which will default to the namespace
+ of the logged-in user.
+
+ It could also be a location inside a bucket, e.g. `buckets///...`
+ config_name (`str`):
+ Configuration name of a dataset. Defaults to "default".
+ set_default (`bool`, *optional*):
+ Whether to set this configuration as the default one. Otherwise, the default configuration is the one
+ named "default".
+ data_dir (`str`, *optional*):
+ Directory name that will contain the uploaded data files. Defaults to the `config_name` if different
+ from "default", else "data".
+
+
+ commit_message (`str`, *optional*):
+ Message to commit while pushing. Will default to `"Upload dataset"`.
+ commit_description (`str`, *optional*):
+ Description of the commit that will be created.
+ Additionally, description of the PR if a PR is created (`create_pr` is True).
+
+
+ private (`bool`, *optional*):
+ Whether to make the repo private. If `None` (default), the repo will be public unless the
+ organization's default is private. This value is ignored if the repo already exists.
+ token (`str`, *optional*):
+ An optional authentication token for the Hugging Face Hub. If no token is passed, will default
+ to the token saved locally when logging in with `huggingface-cli login`. Will raise an error
+ if no token is passed and the user is not logged-in.
+ revision (`str`, *optional*):
+ Branch to push the uploaded files to. Defaults to the `"main"` branch.
+ create_pr (`bool`, *optional*, defaults to `False`):
+ Whether to create a PR with the uploaded files or directly commit.
+ max_shard_size (`int` or `str`, *optional*):
+ Optional maximum size of the dataset shards to be uploaded to the hub. If expressed as a string, needs to be digits followed
+ by a unit (like `"500MB"` or `"1GB"`). If not provided, each split keeps its dataset-native shard count.
+ num_shards (`Dict[str, int]`, *optional*):
+ Number of shards to write. Use a dictionary to define a different `num_shards` for each split.
+ If `max_shard_size` is provided and a split's `num_shards` is not, then the number of shards for that split is estimated
+ from `max_shard_size`.
+ embed_external_files (`bool`, defaults to `True`):
+ Whether to embed file bytes in the shards.
+ In particular, this will do the following before the push for the fields of type:
+
+ - [`Audio`] and [`Image`] removes local path information and embed file content in the Parquet files.
+ num_proc (`int`, *optional*, defaults to `None`):
+ Number of processes when preparing and uploading the dataset.
+ This is helpful if the dataset is made of many samples or media files to embed.
+ I uses "spawn" context to work with hf_xet, the rust client for fast uploads to HF.
+ Multiprocessing is disabled by default.
+
+
+
+ Return:
+ huggingface_hub.CommitInfo
+
+ Example:
+
+ ```python
+ >>> dataset_dict.push_to_hub("/")
+ >>> dataset_dict.push_to_hub("/", private=True)
+ >>> dataset_dict.push_to_hub("/", max_shard_size="1GB")
+ >>> dataset_dict.push_to_hub("/", num_shards={"train": 1024, "test": 8})
+ ```
+
+ If you want to add a new configuration (or subset) to a dataset (e.g. if the dataset has multiple tasks/versions/languages):
+
+ ```python
+ >>> english_dataset.push_to_hub("/", "en")
+ >>> french_dataset.push_to_hub("/", "fr")
+ >>> # later
+ >>> english_dataset = load_dataset("/", "en")
+ >>> french_dataset = load_dataset("/", "fr")
+ ```
+ """
+ # check to make sure that the user doesnt specify the # of shards and max shard sdize at same time, since these are 2 different ways to specify the same thing
+ if max_shard_size is not None and num_shards is not None:
+ raise ValueError(
+ "Failed to push_to_hub: please specify either max_shard_size or num_shards, but not both."
+ )
+
+ if num_shards is None:
+ num_shards = dict.fromkeys(self)
+ elif not isinstance(num_shards, dict):
+ raise ValueError(
+ "Please provide one `num_shards` per dataset in the dataset dictionary, e.g. {{'train': 128, 'test': 4}}"
+ )
+
+ self._check_values_type()
+ self._check_values_features()
+ if config_name == "data":
+ raise ValueError("`config_name` cannot be 'data'. Please, choose another name for configuration.")
+
+ # if max_shard_size is not None and num_shards is not None and any(x is not None for x in num_shards.values()):
+ # raise ValueError(
+ # "Failed to push_to_hub: please specify either max_shard_size or num_shards, but not both."
+ # )
+
+ for split in self:
+ if not re.match(_split_re, split):
+ raise ValueError(f"Split name should match '{_split_re}' but got '{split}'.")
+
+ if not data_dir:
+ data_dir = config_name if config_name != "default" else "data" # for backward compatibility
+
+ api = HfApi(endpoint=config.HF_ENDPOINT, token=token, library_name="datasets", library_version=__version__)
+ if repo_id.startswith("buckets/"):
+ if BucketNotFoundError is None:
+ raise ImportError("Pushing datasets to buckets requires huggingface_hub>=1.6.0")
+ _, _namespace, _bucket_name, *_path_segments = repo_id.split("/")
+ try:
+ bucket_id = api.bucket_info(_namespace + "/" + _bucket_name).id
+ except BucketNotFoundError:
+ bucket_url = api.create_bucket(_namespace + "/" + _bucket_name, private=private, exist_ok=True)
+ bucket_id = bucket_url.bucket_id
+ path = "/".join(s for s in _path_segments if s)
+ return _push_to_bucket(
+ self,
+ bucket_id=bucket_id,
+ path=path,
+ config_name=config_name,
+ set_default=set_default,
+ data_dir=data_dir,
+ token=token,
+ max_shard_size=max_shard_size,
+ num_shards=num_shards.get(split),
+ embed_external_files=embed_external_files,
+ num_proc=num_proc,
+ )
+ else:
+ try:
+ repo_id = api.repo_info(repo_id, repo_type="dataset").id
+ except RepositoryNotFoundError:
+ repo_url = api.create_repo(
+ repo_id,
+ repo_type="dataset",
+ private=private,
+ exist_ok=True,
+ )
+ repo_id = repo_url.repo_id
+
+ if revision is not None and not revision.startswith("refs/pr/"):
+ # We do not call create_branch for a PR reference: 400 Bad Request
+ api.create_branch(repo_id, branch=revision, repo_type="dataset", exist_ok=True)
+ return _push_to_repo(
+ self,
+ repo_id=repo_id,
+ config_name=config_name,
+ set_default=set_default,
+ data_dir=data_dir,
+ commit_message=commit_message,
+ commit_description=commit_description,
+ token=token,
+ create_pr=create_pr,
+ revision=revision,
+ max_shard_size=max_shard_size,
+ num_shards=num_shards,
+ embed_external_files=embed_external_files,
+ num_proc=num_proc,
+ )
+
+
+def _push_to_repo(
+ dset_dict: Union["DatasetDict", "IterableDatasetDict"],
+ repo_id: str,
+ config_name: str = "default",
+ set_default: Optional[bool] = None,
+ data_dir: Optional[str] = None,
+ commit_message: Optional[str] = None,
+ commit_description: Optional[str] = None,
+ token: Optional[str] = None,
+ revision: Optional[str] = None,
+ create_pr: Optional[bool] = False,
+ max_shard_size: Optional[Union[int, str]] = None,
+ num_shards: Optional[dict[str, Optional[int]]] = None,
+ embed_external_files: bool = True,
+ num_proc: Optional[int] = None,
+) -> CommitInfo:
+ api = HfApi(endpoint=config.HF_ENDPOINT, token=token, library_name="datasets", library_version=__version__)
+ resolved_output_path = HfFileSystemResolvedRepositoryPath(
+ repo_id=repo_id, repo_type="dataset", revision=revision or "main", path_in_repo=""
+ )
+
+ additions: list[CommitOperationAdd] = []
+ new_parquet_paths: list[str] = []
+ splits_info: list[SplitInfo] = []
+ uploaded_sizes: list[int] = []
+
+ for split in dset_dict:
+ split_additions, split_new_parquet_paths, features, split_info, uploaded_size = dset_dict[
+ split
+ ]._push_parquet_shards_to_hub(
+ resolved_output_path=resolved_output_path,
+ data_dir=data_dir,
+ split=split,
+ token=token,
+ max_shard_size=max_shard_size,
+ num_shards=(num_shards or {}).get(split),
+ create_pr=create_pr,
+ embed_external_files=embed_external_files,
+ num_proc=num_proc,
+ )
+ additions += split_additions
+ new_parquet_paths += split_new_parquet_paths
+ splits_info.append(split_info)
+ uploaded_sizes.append(uploaded_size)
+
+ commit_message = commit_message if commit_message is not None else "Upload dataset"
+ if len(additions) > config.UPLOADS_MAX_NUMBER_PER_COMMIT:
+ logger.info(
+ f"Number of files to upload is larger than {config.UPLOADS_MAX_NUMBER_PER_COMMIT}. Splitting the push into multiple commits."
+ )
+ num_commits = math.ceil(len(additions) / config.UPLOADS_MAX_NUMBER_PER_COMMIT)
+ for i in range(0, num_commits):
+ operations = additions[
+ i * config.UPLOADS_MAX_NUMBER_PER_COMMIT : (i + 1) * config.UPLOADS_MAX_NUMBER_PER_COMMIT
+ ]
+ for retry, sleep_time in enumerate(itertools.chain(range(10), itertools.repeat(30)), start=1):
+ # We need to retry if another commit happens at the same time
+ sleep_time *= 1 + random.random()
+ try:
+ commit_info = api.create_commit(
+ repo_id,
+ operations=operations,
+ commit_message=commit_message + f" (part {i:05d}-of-{num_commits:05d})",
+ commit_description=commit_description,
+ repo_type="dataset",
+ revision=revision,
+ create_pr=create_pr,
+ )
+ except HfHubHTTPError as err:
+ if (
+ err.__context__
+ and isinstance(err.__context__, HfHubHTTPError)
+ and err.__context__.response.status_code == 409
+ ):
+ # 409 is Conflict (another commit is in progress)
+ time.sleep(sleep_time)
+ logger.info(
+ f"Retrying intermediate commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})"
+ )
+ continue
+ else:
+ raise
+ break
+ logger.info(
+ f"Commit #{i + 1} completed"
+ + (f" (still {num_commits - i - 1} to go)" if num_commits - i - 1 else "")
+ + "."
+ )
+ last_commit_additions = []
+ else:
+ last_commit_additions = additions
+
+ for retry, sleep_time in enumerate(itertools.chain(range(10), itertools.repeat(30)), start=1):
+ # We need to retry if there was a commit in between in case it touched the dataset card data
+ sleep_time *= 1 + random.random()
+
+ # We make sure to get info from this commit
+ parent_commit = api.repo_info(repo_id, repo_type="dataset", revision=revision).sha
+ hf_path = HfFileSystemResolvedRepositoryPath(
+ repo_type=resolved_output_path.repo_type,
+ repo_id=resolved_output_path.repo_id,
+ revision=parent_commit,
+ path_in_repo=resolved_output_path.path_in_repo,
+ ).unresolve()
+ hffs = HfFileSystem(endpoint=config.HF_ENDPOINT, token=token)
+ dirfs = DirFileSystem(fs=hffs, path=hf_path)
+
+ # Check the files to delete
+ try:
+ files_to_delete = list(dirfs.glob(f"{data_dir}/*"))
+ except EntryNotFoundError: # needed for huggingface_hub<=1.7.1
+ files_to_delete = []
+
+ # Don't delete the new files
+ deletions = [
+ CommitOperationDelete(path_in_repo=file_to_delete)
+ for file_to_delete in files_to_delete
+ if file_to_delete not in new_parquet_paths
+ ]
+
+ # Update the dataset card
+ new_dataset_card, new_legacy_dataset_infos = _get_updated_dataset_card(
+ fs=dirfs,
+ config_name=config_name,
+ splits_info=splits_info,
+ features=features,
+ data_dir=data_dir,
+ set_default=set_default,
+ uploaded_sizes=uploaded_sizes,
+ deleted_sizes=[],
+ remove_other_splits=True,
+ )
+ dataset_card_additions = [
+ CommitOperationAdd(path_in_repo=config.REPOCARD_FILENAME, path_or_fileobj=str(new_dataset_card).encode())
+ ]
+ if new_legacy_dataset_infos:
+ dataset_card_additions.append(
+ CommitOperationAdd(
+ path_in_repo=config.DATASETDICT_INFOS_FILENAME,
+ path_or_fileobj=json.dumps(new_legacy_dataset_infos).encode("utf-8"),
+ )
+ )
+ operations = last_commit_additions + dataset_card_additions + deletions
+
+ try:
+ commit_info = api.create_commit(
+ repo_id,
+ operations=operations,
+ commit_message=commit_message,
+ commit_description=commit_description,
+ repo_type="dataset",
+ revision=revision,
+ create_pr=create_pr,
+ parent_commit=parent_commit,
+ )
+ except HfHubHTTPError as err:
+ if (
+ err.__context__
+ and isinstance(err.__context__, HfHubHTTPError)
+ and err.__context__.response.status_code in (412, 409)
+ ):
+ # 412 is Precondition failed (parent_commit isn't satisfied)
+ # 409 is Conflict (another commit is in progress)
+ time.sleep(sleep_time)
+ logger.info(
+ f"Retrying commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})"
+ )
+ continue
+ else:
+ raise
+ break
+
+ return commit_info
+
+
+def _push_to_bucket(
+ dset_dict: Union["DatasetDict", "IterableDatasetDict"],
+ bucket_id: str,
+ path: str,
+ config_name: str = "default",
+ set_default: Optional[bool] = None,
+ data_dir: Optional[str] = None,
+ token: Optional[str] = None,
+ max_shard_size: Optional[Union[int, str]] = None,
+ num_shards: Optional[dict[str, Optional[int]]] = None,
+ embed_external_files: bool = True,
+ num_proc: Optional[int] = None,
+) -> None:
+ api = HfApi(endpoint=config.HF_ENDPOINT, token=token, library_name="datasets", library_version=__version__)
+ resolved_output_path = HfFileSystemResolvedBucketPath(bucket_id=bucket_id, path=path)
+ hf_path = resolved_output_path.unresolve()
+ hffs = HfFileSystem(endpoint=config.HF_ENDPOINT, token=token)
+ dirfs = DirFileSystem(fs=hffs, path=hf_path)
+
+ additions: list[CommitOperationAdd] = []
+ new_parquet_paths: list[str] = []
+ splits_info: list[SplitInfo] = []
+ uploaded_sizes: list[int] = []
+
+ # Check the files to delete before uploading
+ try:
+ files_to_delete = list(dirfs.glob(f"{data_dir}/*"))
+ except EntryNotFoundError: # needed for huggingface_hub<=1.7.1
+ files_to_delete = []
+
+ for split in dset_dict:
+ # Upload the Parquet files
+ split_additions, split_new_parquet_paths, features, split_info, uploaded_size = dset_dict[
+ split
+ ]._push_parquet_shards_to_hub(
+ resolved_output_path=resolved_output_path,
+ data_dir=data_dir,
+ split=split,
+ token=token,
+ max_shard_size=max_shard_size,
+ num_shards=(num_shards or {}).get(split),
+ create_pr=False,
+ embed_external_files=embed_external_files,
+ num_proc=num_proc,
+ )
+ additions += split_additions
+ new_parquet_paths += split_new_parquet_paths
+ splits_info.append(split_info)
+ uploaded_sizes.append(uploaded_size)
+
+ # Don't delete the new files
+ new_parquet_paths = set(new_parquet_paths)
+ delete = [file_to_delete for file_to_delete in files_to_delete if file_to_delete not in new_parquet_paths]
+
+ # Update the dataset card
+ new_dataset_card, new_legacy_dataset_infos = _get_updated_dataset_card(
+ fs=dirfs,
+ config_name=config_name,
+ splits_info=splits_info,
+ features=features,
+ data_dir=data_dir,
+ set_default=set_default,
+ uploaded_sizes=uploaded_sizes,
+ deleted_sizes=[],
+ remove_other_splits=True,
+ )
+ path_prefix = (path + "/") if path else ""
+ add = [(str(new_dataset_card).encode(), path_prefix + config.REPOCARD_FILENAME)]
+ if new_legacy_dataset_infos:
+ add.append(
+ (json.dumps(new_legacy_dataset_infos).encode("utf-8"), path_prefix + config.DATASETDICT_INFOS_FILENAME)
+ )
+
+ # Upload dataset card and delete old files
+ api.batch_bucket_files(
+ bucket_id=bucket_id,
+ add=add,
+ delete=delete,
+ )
diff --git a/venv/Lib/site-packages/datasets/distributed.py b/venv/Lib/site-packages/datasets/distributed.py
new file mode 100644
index 0000000000000000000000000000000000000000..a9ca59c72ff0987e3f06613a64283c79ce83e8db
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/distributed.py
@@ -0,0 +1,43 @@
+from typing import TypeVar
+
+from .arrow_dataset import Dataset, _split_by_node_map_style_dataset
+from .iterable_dataset import IterableDataset, _split_by_node_iterable_dataset
+
+
+DatasetType = TypeVar("DatasetType", Dataset, IterableDataset)
+
+
+def split_dataset_by_node(dataset: DatasetType, rank: int, world_size: int) -> DatasetType:
+ """
+ Split a dataset for the node at rank `rank` in a pool of nodes of size `world_size`.
+
+ For map-style datasets:
+
+ Each node is assigned a chunk of data, e.g. rank 0 is given the first chunk of the dataset.
+ To maximize data loading throughput, chunks are made of contiguous data on disk if possible.
+
+ For iterable datasets:
+
+ If the dataset has a number of shards that is a factor of `world_size` (i.e. if `dataset.num_shards % world_size == 0`),
+ then the shards are evenly assigned across the nodes, which is the most optimized.
+ Otherwise, each node keeps 1 example out of `world_size`, skipping the other examples.
+
+ > [!WARNING]
+ > If you shuffle your iterable dataset in a distributed setup, make sure to set a fixed `seed` in [`IterableDataset.shuffle`]
+ so the same shuffled list of shards is used on every node to know which shards the node should skip.
+
+ Args:
+ dataset ([`Dataset`] or [`IterableDataset`]):
+ The dataset to split by node.
+ rank (`int`):
+ Rank of the current node.
+ world_size (`int`):
+ Total number of nodes.
+
+ Returns:
+ [`Dataset`] or [`IterableDataset`]: The dataset to be used on the node at rank `rank`.
+ """
+ if isinstance(dataset, Dataset):
+ return _split_by_node_map_style_dataset(dataset, rank=rank, world_size=world_size)
+ else:
+ return _split_by_node_iterable_dataset(dataset, rank=rank, world_size=world_size)
diff --git a/venv/Lib/site-packages/datasets/download/__init__.py b/venv/Lib/site-packages/datasets/download/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..8ae0d436504dc2e609b0ca8851509c72a161dbde
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/download/__init__.py
@@ -0,0 +1,10 @@
+__all__ = [
+ "DownloadConfig",
+ "DownloadManager",
+ "DownloadMode",
+ "StreamingDownloadManager",
+]
+
+from .download_config import DownloadConfig
+from .download_manager import DownloadManager, DownloadMode
+from .streaming_download_manager import StreamingDownloadManager
diff --git a/venv/Lib/site-packages/datasets/download/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/download/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..2a843260e396a802a5374e5a8c05c0744d38f913
Binary files /dev/null and b/venv/Lib/site-packages/datasets/download/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/download/__pycache__/download_config.cpython-311.pyc b/venv/Lib/site-packages/datasets/download/__pycache__/download_config.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b07d0dc9bfa8e757c1d3633eaf165cf6464289fd
Binary files /dev/null and b/venv/Lib/site-packages/datasets/download/__pycache__/download_config.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/download/__pycache__/download_manager.cpython-311.pyc b/venv/Lib/site-packages/datasets/download/__pycache__/download_manager.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..cc95dd1b2b45c68e6cdf5159b2a4557d361fab26
Binary files /dev/null and b/venv/Lib/site-packages/datasets/download/__pycache__/download_manager.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/download/__pycache__/streaming_download_manager.cpython-311.pyc b/venv/Lib/site-packages/datasets/download/__pycache__/streaming_download_manager.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b61f32564e372d77c2b6e13e2e3423f816fd24b4
Binary files /dev/null and b/venv/Lib/site-packages/datasets/download/__pycache__/streaming_download_manager.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/download/download_config.py b/venv/Lib/site-packages/datasets/download/download_config.py
new file mode 100644
index 0000000000000000000000000000000000000000..6efae72b671bd0977c52453582dc70561766334b
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/download/download_config.py
@@ -0,0 +1,81 @@
+import copy
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Optional, Union
+
+from .. import config
+
+
+@dataclass
+class DownloadConfig:
+ """Configuration for our cached path manager.
+
+ Attributes:
+ cache_dir (`str` or `Path`, *optional*):
+ Specify a cache directory to save the file to (overwrite the
+ default cache dir).
+ force_download (`bool`, defaults to `False`):
+ If `True`, re-download the file even if it's already cached in
+ the cache dir.
+ resume_download (`bool`, defaults to `False`):
+ If `True`, resume the download if an incompletely received file is
+ found.
+ proxies (`dict`, *optional*):
+ user_agent (`str`, *optional*):
+ Optional string or dict that will be appended to the user-agent on remote
+ requests.
+ extract_compressed_file (`bool`, defaults to `False`):
+ If `True` and the path point to a zip or tar file,
+ extract the compressed file in a folder along the archive.
+ force_extract (`bool`, defaults to `False`):
+ If `True` when `extract_compressed_file` is `True` and the archive
+ was already extracted, re-extract the archive and override the folder where it was extracted.
+ delete_extracted (`bool`, defaults to `False`):
+ Whether to delete (or keep) the extracted files.
+ extract_on_the_fly (`bool`, defaults to `False`):
+ If `True`, extract compressed files while they are being read.
+ use_etag (`bool`, defaults to `True`):
+ Whether to use the ETag HTTP response header to validate the cached files.
+ num_proc (`int`, *optional*):
+ The number of processes to launch to download the files in parallel.
+ max_retries (`int`, default to `1`):
+ The number of times to retry an HTTP request if it fails.
+ token (`str` or `bool`, *optional*):
+ Optional string or boolean to use as Bearer token
+ for remote files on the Datasets Hub. If `True`, or not specified, will get token from `~/.huggingface`.
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the dataset file-system backend, if any.
+ download_desc (`str`, *optional*):
+ A description to be displayed alongside with the progress bar while downloading the files.
+ disable_tqdm (`bool`, defaults to `False`):
+ Whether to disable the individual files download progress bar
+ """
+
+ cache_dir: Optional[Union[str, Path]] = None
+ force_download: bool = False
+ resume_download: bool = False
+ local_files_only: bool = False
+ proxies: Optional[dict] = None
+ user_agent: Optional[str] = None
+ extract_compressed_file: bool = False
+ force_extract: bool = False
+ delete_extracted: bool = False
+ extract_on_the_fly: bool = False
+ use_etag: bool = True
+ num_proc: Optional[int] = None
+ max_retries: int = 1
+ token: Optional[Union[str, bool]] = None
+ storage_options: dict[str, Any] = field(default_factory=dict)
+ download_desc: Optional[str] = None
+ disable_tqdm: bool = False
+
+ def copy(self) -> "DownloadConfig":
+ return self.__class__(**{k: copy.deepcopy(v) for k, v in self.__dict__.items()})
+
+ def __setattr__(self, name, value):
+ if name == "token" and getattr(self, "storage_options", None) is not None:
+ if "hf" not in self.storage_options:
+ self.storage_options["hf"] = {"endpoint": config.HF_ENDPOINT, "token": value}
+ elif getattr(self.storage_options["hf"], "token", None) is None:
+ self.storage_options["hf"]["token"] = value
+ super().__setattr__(name, value)
diff --git a/venv/Lib/site-packages/datasets/download/download_manager.py b/venv/Lib/site-packages/datasets/download/download_manager.py
new file mode 100644
index 0000000000000000000000000000000000000000..b3515bc0e08fd7f809dcbe7dc68dc825aad5f06d
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/download/download_manager.py
@@ -0,0 +1,340 @@
+# Copyright 2020 The TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Download manager interface."""
+
+import enum
+import io
+import multiprocessing
+import os
+from datetime import datetime
+from functools import partial
+from typing import Optional, Union
+
+import fsspec
+from fsspec.core import url_to_fs
+from tqdm.contrib.concurrent import thread_map
+
+from .. import config
+from ..utils import tqdm as hf_tqdm
+from ..utils.file_utils import (
+ ArchiveIterable,
+ FilesIterable,
+ cached_path,
+ is_relative_path,
+ stack_multiprocessing_download_progress_bars,
+ url_or_path_join,
+)
+from ..utils.info_utils import get_size_checksum_dict
+from ..utils.logging import get_logger, tqdm
+from ..utils.py_utils import NestedDataStructure, map_nested
+from ..utils.track import tracked_str
+from .download_config import DownloadConfig
+
+
+logger = get_logger(__name__)
+
+
+class DownloadMode(enum.Enum):
+ """`Enum` for how to treat pre-existing downloads and data.
+
+ The default mode is `REUSE_DATASET_IF_EXISTS`, which will reuse both
+ raw downloads and the prepared dataset if they exist.
+
+ The generations modes:
+
+ | | Downloads | Dataset |
+ |-------------------------------------|-----------|---------|
+ | `REUSE_DATASET_IF_EXISTS` (default) | Reuse | Reuse |
+ | `REUSE_CACHE_IF_EXISTS` | Reuse | Fresh |
+ | `FORCE_REDOWNLOAD` | Fresh | Fresh |
+
+ """
+
+ REUSE_DATASET_IF_EXISTS = "reuse_dataset_if_exists"
+ REUSE_CACHE_IF_EXISTS = "reuse_cache_if_exists"
+ FORCE_REDOWNLOAD = "force_redownload"
+
+
+class DownloadManager:
+ is_streaming = False
+
+ def __init__(
+ self,
+ dataset_name: Optional[str] = None,
+ data_dir: Optional[str] = None,
+ download_config: Optional[DownloadConfig] = None,
+ base_path: Optional[str] = None,
+ record_checksums=False,
+ ):
+ """Download manager constructor.
+
+ Args:
+ data_dir:
+ can be used to specify a manual directory to get the files from.
+ dataset_name (`str`):
+ name of dataset this instance will be used for. If
+ provided, downloads will contain which datasets they were used for.
+ download_config (`DownloadConfig`):
+ to specify the cache directory and other
+ download options
+ base_path (`str`):
+ base path that is used when relative paths are used to
+ download files. This can be a remote url.
+ record_checksums (`bool`, defaults to `False`):
+ Whether to record the checksums of the downloaded files. If None, the value is inferred from the builder.
+ """
+ self._dataset_name = dataset_name
+ self._data_dir = data_dir
+ self._base_path = base_path or os.path.abspath(".")
+ # To record what is being used: {url: {num_bytes: int, checksum: str}}
+ self._recorded_sizes_checksums: dict[str, dict[str, Optional[Union[int, str]]]] = {}
+ self.record_checksums = record_checksums
+ self.download_config = download_config or DownloadConfig()
+ self.downloaded_paths = {}
+ self.extracted_paths = {}
+
+ @property
+ def manual_dir(self):
+ return self._data_dir
+
+ @property
+ def downloaded_size(self):
+ """Returns the total size of downloaded files."""
+ return sum(checksums_dict["num_bytes"] for checksums_dict in self._recorded_sizes_checksums.values())
+
+ def _record_sizes_checksums(self, url_or_urls: NestedDataStructure, downloaded_path_or_paths: NestedDataStructure):
+ """Record size/checksum of downloaded files."""
+ delay = 5
+ for url, path in hf_tqdm(
+ list(zip(url_or_urls.flatten(), downloaded_path_or_paths.flatten())),
+ delay=delay,
+ desc="Computing checksums",
+ ):
+ # call str to support PathLike objects
+ self._recorded_sizes_checksums[str(url)] = get_size_checksum_dict(
+ path, record_checksum=self.record_checksums
+ )
+
+ def download(self, url_or_urls):
+ """Download given URL(s).
+
+ By default, only one process is used for download. Pass customized `download_config.num_proc` to change this behavior.
+
+ Args:
+ url_or_urls (`str` or `list` or `dict`):
+ URL or `list` or `dict` of URLs to download. Each URL is a `str`.
+
+ Returns:
+ `str` or `list` or `dict`:
+ The downloaded paths matching the given input `url_or_urls`.
+
+ Example:
+
+ ```py
+ >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz')
+ ```
+ """
+ download_config = self.download_config.copy()
+ download_config.extract_compressed_file = False
+ if download_config.download_desc is None:
+ download_config.download_desc = "Downloading data"
+
+ download_func = partial(self._download_batched, download_config=download_config)
+
+ start_time = datetime.now()
+ with stack_multiprocessing_download_progress_bars():
+ downloaded_path_or_paths = map_nested(
+ download_func,
+ url_or_urls,
+ map_tuple=True,
+ num_proc=download_config.num_proc,
+ desc="Downloading data files",
+ batched=True,
+ batch_size=-1,
+ )
+ duration = datetime.now() - start_time
+ logger.info(f"Downloading took {duration.total_seconds() // 60} min")
+ url_or_urls = NestedDataStructure(url_or_urls)
+ downloaded_path_or_paths = NestedDataStructure(downloaded_path_or_paths)
+ self.downloaded_paths.update(dict(zip(url_or_urls.flatten(), downloaded_path_or_paths.flatten())))
+
+ start_time = datetime.now()
+ self._record_sizes_checksums(url_or_urls, downloaded_path_or_paths)
+ duration = datetime.now() - start_time
+ logger.info(f"Checksum Computation took {duration.total_seconds() // 60} min")
+
+ return downloaded_path_or_paths.data
+
+ def _download_batched(
+ self,
+ url_or_filenames: list[str],
+ download_config: DownloadConfig,
+ ) -> list[str]:
+ if len(url_or_filenames) >= 16:
+ download_config = download_config.copy()
+ download_config.disable_tqdm = True
+ download_func = partial(self._download_single, download_config=download_config)
+
+ fs: fsspec.AbstractFileSystem
+ path = str(url_or_filenames[0])
+ if is_relative_path(path):
+ # append the relative path to the base_path
+ path = url_or_path_join(self._base_path, path)
+ fs, path = url_to_fs(path, **download_config.storage_options)
+ size = 0
+ try:
+ size = fs.info(path).get("size", 0)
+ except Exception:
+ pass
+ max_workers = (
+ config.HF_DATASETS_MULTITHREADING_MAX_WORKERS if size < (20 << 20) else 1
+ ) # enable multithreading if files are small
+
+ return thread_map(
+ download_func,
+ url_or_filenames,
+ desc=download_config.download_desc or "Downloading",
+ unit="files",
+ position=multiprocessing.current_process()._identity[-1] # contains the ranks of subprocesses
+ if os.environ.get("HF_DATASETS_STACK_MULTIPROCESSING_DOWNLOAD_PROGRESS_BARS") == "1"
+ and multiprocessing.current_process()._identity
+ else None,
+ max_workers=max_workers,
+ tqdm_class=tqdm,
+ )
+ else:
+ return [
+ self._download_single(url_or_filename, download_config=download_config)
+ for url_or_filename in url_or_filenames
+ ]
+
+ def _download_single(self, url_or_filename: str, download_config: DownloadConfig) -> str:
+ url_or_filename = str(url_or_filename)
+ if is_relative_path(url_or_filename):
+ # append the relative path to the base_path
+ url_or_filename = url_or_path_join(self._base_path, url_or_filename)
+ out = cached_path(url_or_filename, download_config=download_config)
+ out = tracked_str(out)
+ out.set_origin(url_or_filename)
+ return out
+
+ def iter_archive(self, path_or_buf: Union[str, io.BufferedReader]):
+ """Iterate over files within an archive.
+
+ Args:
+ path_or_buf (`str` or `io.BufferedReader`):
+ Archive path or archive binary file object.
+
+ Yields:
+ `tuple[str, io.BufferedReader]`:
+ 2-tuple (path_within_archive, file_object).
+ File object is opened in binary mode.
+
+ Example:
+
+ ```py
+ >>> archive = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz')
+ >>> files = dl_manager.iter_archive(archive)
+ ```
+ """
+
+ if hasattr(path_or_buf, "read"):
+ return ArchiveIterable.from_buf(path_or_buf)
+ else:
+ return ArchiveIterable.from_urlpath(path_or_buf)
+
+ def iter_files(self, paths: Union[str, list[str]]):
+ """Iterate over file paths.
+
+ Args:
+ paths (`str` or `list` of `str`):
+ Root paths.
+
+ Yields:
+ `str`: File path.
+
+ Example:
+
+ ```py
+ >>> files = dl_manager.download_and_extract('https://huggingface.co/datasets/AI-Lab-Makerere/beans/resolve/main/data/train.zip')
+ >>> files = dl_manager.iter_files(files)
+ ```
+ """
+ return FilesIterable.from_urlpaths(paths)
+
+ def extract(self, path_or_paths):
+ """Extract given path(s).
+
+ Args:
+ path_or_paths (path or `list` or `dict`):
+ Path of file to extract. Each path is a `str`.
+
+ Returns:
+ extracted_path(s): `str`, The extracted paths matching the given input
+ path_or_paths.
+
+ Example:
+
+ ```py
+ >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz')
+ >>> extracted_files = dl_manager.extract(downloaded_files)
+ ```
+ """
+ download_config = self.download_config.copy()
+ download_config.extract_compressed_file = True
+ extract_func = partial(self._download_single, download_config=download_config)
+ extracted_paths = map_nested(
+ extract_func,
+ path_or_paths,
+ num_proc=download_config.num_proc,
+ desc="Extracting data files",
+ )
+ path_or_paths = NestedDataStructure(path_or_paths)
+ extracted_paths = NestedDataStructure(extracted_paths)
+ self.extracted_paths.update(dict(zip(path_or_paths.flatten(), extracted_paths.flatten())))
+ return extracted_paths.data
+
+ def download_and_extract(self, url_or_urls):
+ """Download and extract given `url_or_urls`.
+
+ Is roughly equivalent to:
+
+ ```
+ extracted_paths = dl_manager.extract(dl_manager.download(url_or_urls))
+ ```
+
+ Args:
+ url_or_urls (`str` or `list` or `dict`):
+ URL or `list` or `dict` of URLs to download and extract. Each URL is a `str`.
+
+ Returns:
+ extracted_path(s): `str`, extracted paths of given URL(s).
+ """
+ return self.extract(self.download(url_or_urls))
+
+ def get_recorded_sizes_checksums(self):
+ return self._recorded_sizes_checksums.copy()
+
+ def delete_extracted_files(self):
+ paths_to_delete = set(self.extracted_paths.values()) - set(self.downloaded_paths.values())
+ for key, path in list(self.extracted_paths.items()):
+ if path in paths_to_delete and os.path.isfile(path):
+ os.remove(path)
+ del self.extracted_paths[key]
+
+ def manage_extracted_files(self):
+ if self.download_config.delete_extracted:
+ self.delete_extracted_files()
diff --git a/venv/Lib/site-packages/datasets/download/streaming_download_manager.py b/venv/Lib/site-packages/datasets/download/streaming_download_manager.py
new file mode 100644
index 0000000000000000000000000000000000000000..bef18148455d782b3dbd65c56d23209b00d369b0
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/download/streaming_download_manager.py
@@ -0,0 +1,219 @@
+import io
+import os
+from collections.abc import Iterable
+from typing import Optional, Union
+
+from ..utils.file_utils import ( # noqa: F401 # backward compatibility
+ SINGLE_FILE_COMPRESSION_PROTOCOLS,
+ ArchiveIterable,
+ FilesIterable,
+ _get_extraction_protocol,
+ _get_path_extension,
+ _prepare_path_and_storage_options,
+ is_relative_path,
+ url_or_path_join,
+ xbasename,
+ xdirname,
+ xet_parse,
+ xexists,
+ xgetsize,
+ xglob,
+ xgzip_open,
+ xisdir,
+ xisfile,
+ xjoin,
+ xlistdir,
+ xnumpy_load,
+ xopen,
+ xpandas_read_csv,
+ xpandas_read_excel,
+ xPath,
+ xpyarrow_parquet_read_table,
+ xrelpath,
+ xsio_loadmat,
+ xsplit,
+ xsplitext,
+ xwalk,
+ xxml_dom_minidom_parse,
+)
+from ..utils.logging import get_logger
+from ..utils.py_utils import map_nested
+from .download_config import DownloadConfig
+
+
+logger = get_logger(__name__)
+
+
+class StreamingDownloadManager:
+ """
+ Download manager that uses the "::" separator to navigate through (possibly remote) compressed archives.
+ Contrary to the regular `DownloadManager`, the `download` and `extract` methods don't actually download nor extract
+ data, but they rather return the path or url that could be opened using the `xopen` function which extends the
+ built-in `open` function to stream data from remote files.
+ """
+
+ is_streaming = True
+
+ def __init__(
+ self,
+ dataset_name: Optional[str] = None,
+ data_dir: Optional[str] = None,
+ download_config: Optional[DownloadConfig] = None,
+ base_path: Optional[str] = None,
+ ):
+ self._dataset_name = dataset_name
+ self._data_dir = data_dir
+ self._base_path = base_path or os.path.abspath(".")
+ self.download_config = download_config or DownloadConfig()
+ self.downloaded_size = None
+ self.record_checksums = False
+
+ @property
+ def manual_dir(self):
+ return self._data_dir
+
+ def download(self, url_or_urls):
+ """Normalize URL(s) of files to stream data from.
+ This is the lazy version of `DownloadManager.download` for streaming.
+
+ Args:
+ url_or_urls (`str` or `list` or `dict`):
+ URL(s) of files to stream data from. Each url is a `str`.
+
+ Returns:
+ url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input url_or_urls.
+
+ Example:
+
+ ```py
+ >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz')
+ ```
+ """
+ url_or_urls = map_nested(self._download_single, url_or_urls, map_tuple=True)
+ return url_or_urls
+
+ def _download_single(self, urlpath: str) -> str:
+ urlpath = str(urlpath)
+ if is_relative_path(urlpath):
+ # append the relative path to the base_path
+ urlpath = url_or_path_join(self._base_path, urlpath)
+ return urlpath
+
+ def extract(self, url_or_urls):
+ """Add extraction protocol for given url(s) for streaming.
+
+ This is the lazy version of `DownloadManager.extract` for streaming.
+
+ Args:
+ url_or_urls (`str` or `list` or `dict`):
+ URL(s) of files to stream data from. Each url is a `str`.
+
+ Returns:
+ url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input `url_or_urls`.
+
+ Example:
+
+ ```py
+ >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz')
+ >>> extracted_files = dl_manager.extract(downloaded_files)
+ ```
+ """
+ urlpaths = map_nested(self._extract, url_or_urls, map_tuple=True)
+ return urlpaths
+
+ def _extract(self, urlpath: str) -> str:
+ urlpath = str(urlpath)
+ # get inner file: zip://train-00000.json.gz::https://foo.bar/data.zip -> zip://train-00000.json.gz
+ protocol = _get_extraction_protocol(urlpath, download_config=self.download_config)
+ path = urlpath.split("::")[0]
+ extension = _get_path_extension(path)
+ if extension in ["tgz", "tar"] or path.endswith((".tar.gz", ".tar.bz2", ".tar.xz")):
+ raise NotImplementedError(
+ f"Extraction protocol for TAR archives like '{urlpath}' is not implemented in streaming mode. "
+ f"Please use `dl_manager.iter_archive` instead.\n\n"
+ f"Example usage:\n\n"
+ f"\turl = dl_manager.download(url)\n"
+ f"\ttar_archive_iterator = dl_manager.iter_archive(url)\n\n"
+ f"\tfor filename, file in tar_archive_iterator:\n"
+ f"\t\t..."
+ )
+ if protocol is None:
+ # no extraction
+ return urlpath
+ elif protocol in SINGLE_FILE_COMPRESSION_PROTOCOLS:
+ # there is one single file which is the uncompressed file
+ inner_file = os.path.basename(urlpath.split("::")[0])
+ inner_file = inner_file[: inner_file.rindex(".")] if "." in inner_file else inner_file
+ return f"{protocol}://{inner_file}::{urlpath}"
+ else:
+ return f"{protocol}://::{urlpath}"
+
+ def download_and_extract(self, url_or_urls):
+ """Prepare given `url_or_urls` for streaming (add extraction protocol).
+
+ This is the lazy version of `DownloadManager.download_and_extract` for streaming.
+
+ Is equivalent to:
+
+ ```
+ urls = dl_manager.extract(dl_manager.download(url_or_urls))
+ ```
+
+ Args:
+ url_or_urls (`str` or `list` or `dict`):
+ URL(s) to stream from data from. Each url is a `str`.
+
+ Returns:
+ url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input `url_or_urls`.
+ """
+ return self.extract(self.download(url_or_urls))
+
+ def iter_archive(self, urlpath_or_buf: Union[str, io.BufferedReader]) -> Iterable[tuple]:
+ """Iterate over files within an archive.
+
+ Args:
+ urlpath_or_buf (`str` or `io.BufferedReader`):
+ Archive path or archive binary file object.
+
+ Yields:
+ `tuple[str, io.BufferedReader]`:
+ 2-tuple (path_within_archive, file_object).
+ File object is opened in binary mode.
+
+ Example:
+
+ ```py
+ >>> archive = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz')
+ >>> files = dl_manager.iter_archive(archive)
+ ```
+ """
+
+ if hasattr(urlpath_or_buf, "read"):
+ return ArchiveIterable.from_buf(urlpath_or_buf)
+ else:
+ return ArchiveIterable.from_urlpath(urlpath_or_buf, download_config=self.download_config)
+
+ def iter_files(self, urlpaths: Union[str, list[str]]) -> Iterable[str]:
+ """Iterate over files.
+
+ Args:
+ urlpaths (`str` or `list` of `str`):
+ Root paths.
+
+ Yields:
+ str: File URL path.
+
+ Example:
+
+ ```py
+ >>> files = dl_manager.download_and_extract('https://huggingface.co/datasets/AI-Lab-Makerere/beans/resolve/main/data/train.zip')
+ >>> files = dl_manager.iter_files(files)
+ ```
+ """
+ return FilesIterable.from_urlpaths(urlpaths, download_config=self.download_config)
+
+ def manage_extracted_files(self):
+ pass
+
+ def get_recorded_sizes_checksums(self):
+ pass
diff --git a/venv/Lib/site-packages/datasets/exceptions.py b/venv/Lib/site-packages/datasets/exceptions.py
new file mode 100644
index 0000000000000000000000000000000000000000..7c1724fb572dc98a2b65ad07f0a1a08c5d2a32e5
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/exceptions.py
@@ -0,0 +1,119 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2023 The HuggingFace Authors.
+from typing import Any, Optional, Union
+
+from huggingface_hub import HfFileSystem
+
+from . import config
+from .table import CastError
+from .utils.track import TrackedIterableFromGenerator, tracked_list, tracked_str
+
+
+class DatasetsError(Exception):
+ """Base class for exceptions in this library."""
+
+
+class DefunctDatasetError(DatasetsError):
+ """The dataset has been defunct."""
+
+
+class FileNotFoundDatasetsError(DatasetsError, FileNotFoundError):
+ """FileNotFoundError raised by this library."""
+
+
+class DataFilesNotFoundError(FileNotFoundDatasetsError):
+ """No (supported) data files found."""
+
+
+class DatasetNotFoundError(FileNotFoundDatasetsError):
+ """Dataset not found.
+
+ Raised when trying to access:
+ - a missing dataset, or
+ - a private/gated dataset and the user is not authenticated.
+ """
+
+
+class DatasetBuildError(DatasetsError):
+ pass
+
+
+class ManualDownloadError(DatasetBuildError):
+ pass
+
+
+class FileFormatError(DatasetBuildError):
+ pass
+
+
+class DatasetGenerationError(DatasetBuildError):
+ pass
+
+
+class DatasetGenerationCastError(DatasetGenerationError):
+ @classmethod
+ def from_cast_error(
+ cls,
+ cast_error: CastError,
+ builder_name: str,
+ gen_kwargs: dict[str, Any],
+ token: Optional[Union[bool, str]],
+ ) -> "DatasetGenerationCastError":
+ explanation_message = (
+ f"\n\nAll the data files must have the same columns, but at some point {cast_error.details()}"
+ )
+ formatted_tracked_gen_kwargs: list[str] = []
+ for gen_kwarg in gen_kwargs.values():
+ if not isinstance(gen_kwarg, (tracked_str, tracked_list, TrackedIterableFromGenerator)):
+ continue
+ while (
+ isinstance(gen_kwarg, (tracked_list, TrackedIterableFromGenerator)) and gen_kwarg.last_item is not None
+ ):
+ gen_kwarg = gen_kwarg.last_item
+ if isinstance(gen_kwarg, tracked_str):
+ gen_kwarg = gen_kwarg.get_origin()
+ if isinstance(gen_kwarg, str) and gen_kwarg.startswith("hf://"):
+ resolved_path = HfFileSystem(endpoint=config.HF_ENDPOINT, token=token).resolve_path(gen_kwarg)
+ gen_kwarg = "hf://" + resolved_path.unresolve()
+ if "@" + resolved_path.revision in gen_kwarg:
+ gen_kwarg = (
+ gen_kwarg.replace("@" + resolved_path.revision, "", 1)
+ + f" (at revision {resolved_path.revision})"
+ )
+ formatted_tracked_gen_kwargs.append(str(gen_kwarg))
+ if formatted_tracked_gen_kwargs:
+ explanation_message += f"\n\nThis happened while the {builder_name} dataset builder was generating data using\n\n{', '.join(formatted_tracked_gen_kwargs)}"
+ help_message = "\n\nPlease either edit the data files to have matching columns, or separate them into different configurations (see docs at https://hf.co/docs/hub/datasets-manual-configuration#multiple-configurations)"
+ return cls("An error occurred while generating the dataset" + explanation_message + help_message)
+
+
+class ChecksumVerificationError(DatasetsError):
+ """Error raised during checksums verifications of downloaded files."""
+
+
+class UnexpectedDownloadedFileError(ChecksumVerificationError):
+ """Some downloaded files were not expected."""
+
+
+class ExpectedMoreDownloadedFilesError(ChecksumVerificationError):
+ """Some files were supposed to be downloaded but were not."""
+
+
+class NonMatchingChecksumError(ChecksumVerificationError):
+ """The downloaded file checksum don't match the expected checksum."""
+
+
+class SplitsVerificationError(DatasetsError):
+ """Error raised during splits verifications."""
+
+
+class UnexpectedSplitsError(SplitsVerificationError):
+ """The expected splits of the downloaded file is missing."""
+
+
+class ExpectedMoreSplitsError(SplitsVerificationError):
+ """Some recorded splits are missing."""
+
+
+class NonMatchingSplitsSizesError(SplitsVerificationError):
+ """The splits sizes don't match the expected splits sizes."""
diff --git a/venv/Lib/site-packages/datasets/features/__init__.py b/venv/Lib/site-packages/datasets/features/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..163aaa0e2c516b81d5c3fc9d202760903d3e72ba
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/features/__init__.py
@@ -0,0 +1,29 @@
+__all__ = [
+ "Audio",
+ "Array2D",
+ "Array3D",
+ "Array4D",
+ "Array5D",
+ "ClassLabel",
+ "Features",
+ "Json",
+ "LargeList",
+ "List",
+ "Sequence",
+ "Value",
+ "Image",
+ "Mesh",
+ "Translation",
+ "TranslationVariableLanguages",
+ "Video",
+ "Pdf",
+ "Nifti",
+]
+from .audio import Audio
+from .features import Array2D, Array3D, Array4D, Array5D, ClassLabel, Features, Json, LargeList, List, Sequence, Value
+from .image import Image
+from .mesh import Mesh
+from .nifti import Nifti
+from .pdf import Pdf
+from .translation import Translation, TranslationVariableLanguages
+from .video import Video
diff --git a/venv/Lib/site-packages/datasets/features/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/features/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..bd62006fe9d7adbefd6476446ce7febbf4e49c24
Binary files /dev/null and b/venv/Lib/site-packages/datasets/features/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/features/__pycache__/_torchcodec.cpython-311.pyc b/venv/Lib/site-packages/datasets/features/__pycache__/_torchcodec.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b38f948d2ea7001e1cd76377426314e7516f86f3
Binary files /dev/null and b/venv/Lib/site-packages/datasets/features/__pycache__/_torchcodec.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/features/__pycache__/audio.cpython-311.pyc b/venv/Lib/site-packages/datasets/features/__pycache__/audio.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..79b7efbcac062840f5d11be09bb54d1809367d93
Binary files /dev/null and b/venv/Lib/site-packages/datasets/features/__pycache__/audio.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/features/__pycache__/features.cpython-311.pyc b/venv/Lib/site-packages/datasets/features/__pycache__/features.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d5be329d2a3638951c49f3a1774f4c72205e6ebd
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/features/__pycache__/features.cpython-311.pyc
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d21f1462a8a67aee40cc7241776b97ada53a16210336abdfb63e4ea69617821
+size 143679
diff --git a/venv/Lib/site-packages/datasets/features/__pycache__/image.cpython-311.pyc b/venv/Lib/site-packages/datasets/features/__pycache__/image.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6418229d2ed766309e872a96aeef8d819a26a224
Binary files /dev/null and b/venv/Lib/site-packages/datasets/features/__pycache__/image.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/features/__pycache__/mesh.cpython-311.pyc b/venv/Lib/site-packages/datasets/features/__pycache__/mesh.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..62aa5be48d9c33cc7d5694b9bdae02765b3e67f4
Binary files /dev/null and b/venv/Lib/site-packages/datasets/features/__pycache__/mesh.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/features/__pycache__/nifti.cpython-311.pyc b/venv/Lib/site-packages/datasets/features/__pycache__/nifti.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..313253420138a266cbb68be705c2acbc43de9d09
Binary files /dev/null and b/venv/Lib/site-packages/datasets/features/__pycache__/nifti.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/features/__pycache__/pdf.cpython-311.pyc b/venv/Lib/site-packages/datasets/features/__pycache__/pdf.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..fe6191dfbd2975a59f2e40e57877e56706e3880f
Binary files /dev/null and b/venv/Lib/site-packages/datasets/features/__pycache__/pdf.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/features/__pycache__/translation.cpython-311.pyc b/venv/Lib/site-packages/datasets/features/__pycache__/translation.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..978e633a1cafc11a14d06523dfd242de8c7662e3
Binary files /dev/null and b/venv/Lib/site-packages/datasets/features/__pycache__/translation.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/features/__pycache__/video.cpython-311.pyc b/venv/Lib/site-packages/datasets/features/__pycache__/video.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c043a598c45a6b7578bddc3c338a7dd999ab7229
Binary files /dev/null and b/venv/Lib/site-packages/datasets/features/__pycache__/video.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/features/_torchcodec.py b/venv/Lib/site-packages/datasets/features/_torchcodec.py
new file mode 100644
index 0000000000000000000000000000000000000000..d71b4156a5124be4a6433ce76a3794d55393499c
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/features/_torchcodec.py
@@ -0,0 +1,15 @@
+import numpy as np
+from torchcodec.decoders import AudioDecoder as _AudioDecoder
+
+
+class AudioDecoder(_AudioDecoder):
+ def __getitem__(self, key: str):
+ if key == "array":
+ y = self.get_all_samples().data.cpu().numpy()
+ return np.mean(y, axis=tuple(range(y.ndim - 1))) if y.ndim > 1 else y
+ elif key == "sampling_rate":
+ return self.get_samples_played_in_range(0, 0).sample_rate
+ elif hasattr(super(), "__getitem__"):
+ return super().__getitem__(key)
+ else:
+ raise TypeError("'torchcodec.decoders.AudioDecoder' object is not subscriptable")
diff --git a/venv/Lib/site-packages/datasets/features/audio.py b/venv/Lib/site-packages/datasets/features/audio.py
new file mode 100644
index 0000000000000000000000000000000000000000..ac2ddeef67cd1299f137c8f6d3c48d8c67229093
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/features/audio.py
@@ -0,0 +1,365 @@
+import os
+from dataclasses import dataclass, field
+from io import BytesIO
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union
+
+import numpy as np
+import pyarrow as pa
+
+from .. import config
+from ..download.download_config import DownloadConfig
+from ..table import array_cast
+from ..utils.file_utils import is_local_path, is_remote_url, xopen
+from ..utils.py_utils import no_op_if_value_is_null, string_to_dict
+
+
+if TYPE_CHECKING:
+ from torchcodec.decoders import AudioDecoder
+
+ from .features import FeatureType
+
+
+@dataclass
+class Audio:
+ """Audio [`Feature`] to extract audio data from an audio file.
+
+ Input: The Audio feature accepts as input:
+ - A `str`: Absolute path to the audio file (i.e. random access is allowed).
+ - A `pathlib.Path`: path to the audio file (i.e. random access is allowed).
+ - A `dict` with the keys:
+
+ - `path`: String with relative path of the audio file to the archive file.
+ - `bytes`: Bytes content of the audio file.
+
+ This is useful for parquet or webdataset files which embed audio files.
+
+ - A `dict` with the keys:
+
+ - `array`: Array containing the audio sample
+ - `sampling_rate`: Integer corresponding to the sampling rate of the audio sample.
+
+ - A `torchcodec.decoders.AudioDecoder`: torchcodec audio decoder object.
+
+ Output: The Audio features output data as `torchcodec.decoders.AudioDecoder` objects, with additional keys:
+
+ - `array`: Array containing the audio sample
+ - `sampling_rate`: Integer corresponding to the sampling rate of the audio sample.
+
+ Args:
+ sampling_rate (`int`, *optional*):
+ Target sampling rate. If `None`, the native sampling rate is used.
+ num_channels (`int`, *optional*):
+ The desired number of channels of the samples. By default, the number of channels of the source is used.
+ Audio decoding will return samples with shape (num_channels, num_samples)
+ Currently `None` (number of channels of the source, default), `1` (mono) or `2` (stereo) channels are supported.
+ The `num_channels` argument is passed to `torchcodec.decoders.AudioDecoder`.
+
+
+ decode (`bool`, defaults to `True`):
+ Whether to decode the audio data. If `False`,
+ returns the underlying dictionary in the format `{"path": audio_path, "bytes": audio_bytes}`.
+ stream_index (`int`, *optional*):
+ The streaming index to use from the file. If `None` defaults to the "best" index.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset, Audio
+ >>> ds = load_dataset("PolyAI/minds14", name="en-US", split="train")
+ >>> ds = ds.cast_column("audio", Audio(sampling_rate=44100, num_channels=2))
+ >>> ds[0]["audio"]
+
+ >>> audio = ds[0]["audio"]
+ >>> audio.get_samples_played_in_range(0, 10)
+ AudioSamples:
+ data (shape): torch.Size([2, 110592])
+ pts_seconds: 0.0
+ duration_seconds: 2.507755102040816
+ sample_rate: 44100
+ ```
+ """
+
+ sampling_rate: Optional[int] = None
+ decode: bool = True
+ num_channels: Optional[int] = None
+ stream_index: Optional[int] = None
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ dtype: ClassVar[str] = "dict"
+ pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()})
+ _type: str = field(default="Audio", init=False, repr=False)
+
+ def __call__(self):
+ return self.pa_type
+
+ def encode_example(self, value: Union[str, bytes, bytearray, dict, "AudioDecoder"]) -> dict:
+ """Encode example into a format for Arrow.
+
+ Args:
+ value (`str`, `bytes`,`bytearray`,`dict`, `AudioDecoder`):
+ Data passed as input to Audio feature.
+
+ Returns:
+ `dict`
+ """
+ try:
+ import torch
+ from torchcodec.encoders import AudioEncoder # needed to write audio files
+ except ImportError as err:
+ raise ImportError("To support encoding audio data, please install 'torchcodec'.") from err
+
+ if value is None:
+ raise ValueError("value must be provided")
+
+ if config.TORCHCODEC_AVAILABLE:
+ from torchcodec.decoders import AudioDecoder
+
+ else:
+ AudioDecoder = None
+
+ if isinstance(value, str):
+ return {"bytes": None, "path": value}
+ elif isinstance(value, Path):
+ return {"bytes": None, "path": str(value.absolute())}
+ elif isinstance(value, (bytes, bytearray)):
+ return {"bytes": value, "path": None}
+ elif AudioDecoder is not None and isinstance(value, AudioDecoder):
+ return encode_torchcodec_audio(value)
+ elif "array" in value:
+ # convert the audio array to wav bytes
+ buffer = BytesIO()
+ AudioEncoder(
+ torch.from_numpy(value["array"].astype(np.float32)), sample_rate=value["sampling_rate"]
+ ).to_file_like(buffer, format="wav", num_channels=self.num_channels)
+ return {"bytes": buffer.getvalue(), "path": None}
+ elif value.get("path") is not None and os.path.isfile(value["path"]):
+ # we set "bytes": None to not duplicate the data if they're already available locally
+ if value["path"].endswith("pcm"):
+ # "PCM" only has raw audio bytes
+ if value.get("sampling_rate") is None:
+ # At least, If you want to convert "PCM-byte" to "WAV-byte", you have to know sampling rate
+ raise KeyError("To use PCM files, please specify a 'sampling_rate' in Audio object")
+ if value.get("bytes"):
+ # If we already had PCM-byte, we don`t have to make "read file, make bytes" (just use it!)
+ bytes_value = np.frombuffer(value["bytes"], dtype=np.int16).astype(np.float32) / 32767
+ else:
+ bytes_value = np.memmap(value["path"], dtype="h", mode="r").astype(np.float32) / 32767
+
+ buffer = BytesIO()
+ AudioEncoder(torch.from_numpy(bytes_value), sample_rate=value["sampling_rate"]).to_file_like(
+ buffer, format="wav", num_channels=self.num_channels
+ )
+ return {"bytes": buffer.getvalue(), "path": None}
+ else:
+ return {"bytes": None, "path": value.get("path")}
+ elif value.get("bytes") is not None or value.get("path") is not None:
+ # store the audio bytes, and path is used to infer the audio format using the file extension
+ return {"bytes": value.get("bytes"), "path": value.get("path")}
+ else:
+ raise ValueError(
+ f"An audio sample should have one of 'path' or 'bytes' but they are missing or None in {value}."
+ )
+
+ def decode_example(
+ self, value: dict, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None
+ ) -> "AudioDecoder":
+ """Decode example audio file into audio data.
+
+ Args:
+ value (`dict`):
+ A dictionary with keys:
+
+ - `path`: String with relative audio file path.
+ - `bytes`: Bytes of the audio file.
+ token_per_repo_id (`dict`, *optional*):
+ To access and decode
+ audio files from private repositories on the Hub, you can pass
+ a dictionary repo_id (`str`) -> token (`bool` or `str`)
+
+ Returns:
+ `torchcodec.decoders.AudioDecoder`
+ """
+ if config.TORCHCODEC_AVAILABLE:
+ from ._torchcodec import AudioDecoder
+ else:
+ raise ImportError("To support decoding audio data, please install 'torchcodec'.")
+
+ if not self.decode:
+ raise RuntimeError("Decoding is disabled for this feature. Please use Audio(decode=True) instead.")
+
+ path, bytes = (value["path"], value["bytes"]) if value["bytes"] is not None else (value["path"], None)
+ if path is None and bytes is None:
+ raise ValueError(f"An audio sample should have one of 'path' or 'bytes' but both are None in {value}.")
+
+ if bytes is None and is_local_path(path):
+ audio = AudioDecoder(
+ path, stream_index=self.stream_index, sample_rate=self.sampling_rate, num_channels=self.num_channels
+ )
+
+ elif bytes is None:
+ token_per_repo_id = token_per_repo_id or {}
+ source_url = path.split("::")[-1]
+ pattern = (
+ config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL
+ )
+ source_url_fields = string_to_dict(source_url, pattern)
+ token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None
+
+ download_config = DownloadConfig(token=token)
+ f = xopen(path, "rb", download_config=download_config)
+ audio = AudioDecoder(
+ f, stream_index=self.stream_index, sample_rate=self.sampling_rate, num_channels=self.num_channels
+ )
+
+ else:
+ audio = AudioDecoder(
+ bytes, stream_index=self.stream_index, sample_rate=self.sampling_rate, num_channels=self.num_channels
+ )
+ audio._hf_encoded = {"path": path, "bytes": bytes}
+ audio.metadata.path = path
+ return audio
+
+ def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]:
+ """If in the decodable state, raise an error, otherwise flatten the feature into a dictionary."""
+ from .features import Value
+
+ if self.decode:
+ raise ValueError("Cannot flatten a decoded Audio feature.")
+ return {
+ "bytes": Value("binary"),
+ "path": Value("string"),
+ }
+
+ def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray]) -> pa.StructArray:
+ """Cast an Arrow array to the Audio arrow storage type.
+ The Arrow types that can be converted to the Audio pyarrow storage type are:
+
+ - `pa.string()` - it must contain the "path" data
+ - `pa.binary()` - it must contain the audio bytes
+ - `pa.struct({"bytes": pa.binary()})`
+ - `pa.struct({"path": pa.string()})`
+ - `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter
+
+ Args:
+ storage (`Union[pa.StringArray, pa.StructArray]`):
+ PyArrow array to cast.
+
+ Returns:
+ `pa.StructArray`: Array in the Audio arrow storage type, that is
+ `pa.struct({"bytes": pa.binary(), "path": pa.string()})`
+ """
+ if pa.types.is_string(storage.type):
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
+ storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_large_binary(storage.type):
+ storage = array_cast(
+ storage, pa.binary()
+ ) # this can fail in case of big audios, paths should be used instead
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_binary(storage.type):
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_struct(storage.type) and storage.type.get_all_field_indices("array"):
+ storage = pa.array(
+ [Audio().encode_example(x) if x is not None else None for x in storage.to_numpy(zero_copy_only=False)]
+ )
+ elif pa.types.is_struct(storage.type):
+ if storage.type.get_field_index("bytes") >= 0:
+ bytes_array = storage.field("bytes")
+ else:
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
+ if storage.type.get_field_index("path") >= 0:
+ path_array = storage.field("path")
+ else:
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null())
+ return array_cast(storage, self.pa_type)
+
+ def embed_storage(
+ self, storage: pa.StructArray, token_per_repo_id=None, local_files: bool = True, remote_files: bool = True
+ ) -> pa.StructArray:
+ """Embed audio files into the Arrow array.
+
+ Args:
+ storage (`pa.StructArray`):
+ PyArrow array to embed.
+ token_per_repo_id (`dict`, optional):
+ Dictionary repo_id -> token to fetch the files bytes.
+ local_files (`bool`, defaults to `True`)
+ Whether to embed local files data in the array
+
+
+ remote_files (`bool`, defaults to `True`)
+ Whether to embed remote files data in the array.
+ E.g. files with paths that start with hf:// or https://
+
+
+
+ Returns:
+ `pa.StructArray`: Array in the Audio arrow storage type, that is
+ `pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
+ """
+ if token_per_repo_id is None:
+ token_per_repo_id = {}
+
+ @no_op_if_value_is_null
+ def path_to_bytes(path):
+ source_url = path.split("::")[-1]
+ pattern = (
+ config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL
+ )
+ source_url_fields = string_to_dict(source_url, pattern)
+ token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None
+ download_config = DownloadConfig(token=token)
+ with xopen(path, "rb", download_config=download_config) as f:
+ return f.read()
+
+ bytes_array = pa.array(
+ [
+ (
+ path_to_bytes(x["path"])
+ if x["bytes"] is None
+ and ((local_files and is_local_path(x["path"])) or (remote_files and is_remote_url(x["path"])))
+ else x["bytes"]
+ )
+ if x is not None
+ else None
+ for x in storage.to_pylist()
+ ],
+ type=pa.binary(),
+ )
+ path_array = pa.array(
+ [
+ (
+ os.path.basename(path)
+ if (local_files and is_local_path(path)) or (remote_files and is_remote_url(path))
+ else path
+ )
+ if path is not None
+ else None
+ for path in storage.field("path").to_pylist()
+ ],
+ type=pa.string(),
+ )
+ storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null())
+ return array_cast(storage, self.pa_type)
+
+
+def encode_torchcodec_audio(audio: "AudioDecoder") -> dict:
+ if hasattr(audio, "_hf_encoded"):
+ return audio._hf_encoded
+ else:
+ try:
+ from torchcodec.encoders import AudioEncoder # needed to write audio files
+ except ImportError as err:
+ raise ImportError("To support encoding audio data, please install 'torchcodec'.") from err
+
+ samples = audio.get_all_samples()
+ buffer = BytesIO()
+ num_channels = samples.data.shape[0]
+ AudioEncoder(samples.data.cpu(), sample_rate=samples.sample_rate).to_file_like(
+ buffer, format="wav", num_channels=num_channels
+ )
+ return {"bytes": buffer.getvalue(), "path": None}
diff --git a/venv/Lib/site-packages/datasets/features/features.py b/venv/Lib/site-packages/datasets/features/features.py
new file mode 100644
index 0000000000000000000000000000000000000000..17b3d441960c242ba6b9adbbe753cd185742465a
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/features/features.py
@@ -0,0 +1,2449 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""This class handle features definition in datasets and some utilities to display table type."""
+
+import copy
+import json
+import re
+import sys
+from collections.abc import Iterable, Mapping
+from collections.abc import Sequence as SequenceABC
+from collections.abc import Sequence as Sequence_
+from dataclasses import InitVar, dataclass, field, fields
+from functools import reduce, wraps
+from operator import mul
+from typing import Any, Callable, ClassVar, Literal, Optional, Union
+
+import numpy as np
+import pandas as pd
+import pyarrow as pa
+import pyarrow.compute as pc
+import pyarrow.types
+from pandas.api.extensions import ExtensionArray as PandasExtensionArray
+from pandas.api.extensions import ExtensionDtype as PandasExtensionDtype
+
+from .. import config
+from ..naming import camelcase_to_snakecase, snakecase_to_camelcase
+from ..table import array_cast
+from ..utils import experimental, logging
+from ..utils.json import ujson_dumps, ujson_loads
+from ..utils.py_utils import asdict, first_non_null_value, zip_dict
+from .audio import Audio
+from .image import Image, encode_pil_image
+from .mesh import Mesh
+from .nifti import Nifti, encode_nibabel_image
+from .pdf import Pdf, encode_pdfplumber_pdf
+from .translation import Translation, TranslationVariableLanguages
+from .video import Video
+
+
+logger = logging.get_logger(__name__)
+
+
+def _arrow_to_datasets_dtype(arrow_type: pa.DataType) -> str:
+ """
+ _arrow_to_datasets_dtype takes a pyarrow.DataType and converts it to a datasets string dtype.
+ In effect, `dt == string_to_arrow(_arrow_to_datasets_dtype(dt))`
+ """
+ if pyarrow.types.is_null(arrow_type):
+ return "null"
+ elif pyarrow.types.is_boolean(arrow_type):
+ return "bool"
+ elif pyarrow.types.is_int8(arrow_type):
+ return "int8"
+ elif pyarrow.types.is_int16(arrow_type):
+ return "int16"
+ elif pyarrow.types.is_int32(arrow_type):
+ return "int32"
+ elif pyarrow.types.is_int64(arrow_type):
+ return "int64"
+ elif pyarrow.types.is_uint8(arrow_type):
+ return "uint8"
+ elif pyarrow.types.is_uint16(arrow_type):
+ return "uint16"
+ elif pyarrow.types.is_uint32(arrow_type):
+ return "uint32"
+ elif pyarrow.types.is_uint64(arrow_type):
+ return "uint64"
+ elif pyarrow.types.is_float16(arrow_type):
+ return "float16" # pyarrow dtype is "halffloat"
+ elif pyarrow.types.is_float32(arrow_type):
+ return "float32" # pyarrow dtype is "float"
+ elif pyarrow.types.is_float64(arrow_type):
+ return "float64" # pyarrow dtype is "double"
+ elif pyarrow.types.is_time32(arrow_type):
+ return f"time32[{pa.type_for_alias(str(arrow_type)).unit}]"
+ elif pyarrow.types.is_time64(arrow_type):
+ return f"time64[{pa.type_for_alias(str(arrow_type)).unit}]"
+ elif pyarrow.types.is_timestamp(arrow_type):
+ if arrow_type.tz is None:
+ return f"timestamp[{arrow_type.unit}]"
+ elif arrow_type.tz:
+ return f"timestamp[{arrow_type.unit}, tz={arrow_type.tz}]"
+ else:
+ raise ValueError(f"Unexpected timestamp object {arrow_type}.")
+ elif pyarrow.types.is_date32(arrow_type):
+ return "date32" # pyarrow dtype is "date32[day]"
+ elif pyarrow.types.is_date64(arrow_type):
+ return "date64" # pyarrow dtype is "date64[ms]"
+ elif pyarrow.types.is_duration(arrow_type):
+ return f"duration[{arrow_type.unit}]"
+ elif pyarrow.types.is_decimal128(arrow_type):
+ return f"decimal128({arrow_type.precision}, {arrow_type.scale})"
+ elif pyarrow.types.is_decimal256(arrow_type):
+ return f"decimal256({arrow_type.precision}, {arrow_type.scale})"
+ elif pyarrow.types.is_binary(arrow_type):
+ return "binary"
+ elif pyarrow.types.is_large_binary(arrow_type):
+ return "large_binary"
+ elif pyarrow.types.is_binary_view(arrow_type):
+ return "binary_view"
+ elif pyarrow.types.is_string(arrow_type):
+ return "string"
+ elif pyarrow.types.is_large_string(arrow_type):
+ return "large_string"
+ elif pyarrow.types.is_string_view(arrow_type):
+ return "string_view"
+ elif pyarrow.types.is_dictionary(arrow_type):
+ return _arrow_to_datasets_dtype(arrow_type.value_type)
+ else:
+ raise ValueError(f"Arrow type {arrow_type} does not have a datasets dtype equivalent.")
+
+
+def string_to_arrow(datasets_dtype: str) -> pa.DataType:
+ """
+ string_to_arrow takes a datasets string dtype and converts it to a pyarrow.DataType.
+
+ In effect, `dt == string_to_arrow(_arrow_to_datasets_dtype(dt))`
+
+ This is necessary because the datasets.Value() primitive type is constructed using a string dtype
+
+ Value(dtype=str)
+
+ But Features.type (via `get_nested_type()` expects to resolve Features into a pyarrow Schema,
+ which means that each Value() must be able to resolve into a corresponding pyarrow.DataType, which is the
+ purpose of this function.
+ """
+ if datasets_dtype == "json":
+ raise ValueError("'json' is not a valid dtype, use the Json() feature instead")
+
+ def _dtype_error_msg(dtype, pa_dtype, examples=None, urls=None):
+ msg = f"{dtype} is not a validly formatted string representation of the pyarrow {pa_dtype} type."
+ if examples:
+ examples = ", ".join(examples[:-1]) + " or " + examples[-1] if len(examples) > 1 else examples[0]
+ msg += f"\nValid examples include: {examples}."
+ if urls:
+ urls = ", ".join(urls[:-1]) + " and " + urls[-1] if len(urls) > 1 else urls[0]
+ msg += f"\nFor more insformation, see: {urls}."
+ return msg
+
+ if datasets_dtype in pa.__dict__:
+ return pa.__dict__[datasets_dtype]()
+
+ if (datasets_dtype + "_") in pa.__dict__:
+ return pa.__dict__[datasets_dtype + "_"]()
+
+ timestamp_matches = re.search(r"^timestamp\[(.*)\]$", datasets_dtype)
+ if timestamp_matches:
+ timestamp_internals = timestamp_matches.group(1)
+ internals_matches = re.search(r"^(s|ms|us|ns),\s*tz=([a-zA-Z0-9/_+\-:]*)$", timestamp_internals)
+ if timestamp_internals in ["s", "ms", "us", "ns"]:
+ return pa.timestamp(timestamp_internals)
+ elif internals_matches:
+ return pa.timestamp(internals_matches.group(1), internals_matches.group(2))
+ else:
+ raise ValueError(
+ _dtype_error_msg(
+ datasets_dtype,
+ "timestamp",
+ examples=["timestamp[us]", "timestamp[us, tz=America/New_York"],
+ urls=["https://arrow.apache.org/docs/python/generated/pyarrow.timestamp.html"],
+ )
+ )
+
+ duration_matches = re.search(r"^duration\[(.*)\]$", datasets_dtype)
+ if duration_matches:
+ duration_internals = duration_matches.group(1)
+ if duration_internals in ["s", "ms", "us", "ns"]:
+ return pa.duration(duration_internals)
+ else:
+ raise ValueError(
+ _dtype_error_msg(
+ datasets_dtype,
+ "duration",
+ examples=["duration[s]", "duration[us]"],
+ urls=["https://arrow.apache.org/docs/python/generated/pyarrow.duration.html"],
+ )
+ )
+
+ time_matches = re.search(r"^time(.*)\[(.*)\]$", datasets_dtype)
+ if time_matches:
+ time_internals_bits = time_matches.group(1)
+ if time_internals_bits == "32":
+ time_internals_unit = time_matches.group(2)
+ if time_internals_unit in ["s", "ms"]:
+ return pa.time32(time_internals_unit)
+ else:
+ raise ValueError(
+ f"{time_internals_unit} is not a valid unit for the pyarrow time32 type. Supported units: s (second) and ms (millisecond)."
+ )
+ elif time_internals_bits == "64":
+ time_internals_unit = time_matches.group(2)
+ if time_internals_unit in ["us", "ns"]:
+ return pa.time64(time_internals_unit)
+ else:
+ raise ValueError(
+ f"{time_internals_unit} is not a valid unit for the pyarrow time64 type. Supported units: us (microsecond) and ns (nanosecond)."
+ )
+ else:
+ raise ValueError(
+ _dtype_error_msg(
+ datasets_dtype,
+ "time",
+ examples=["time32[s]", "time64[us]"],
+ urls=[
+ "https://arrow.apache.org/docs/python/generated/pyarrow.time32.html",
+ "https://arrow.apache.org/docs/python/generated/pyarrow.time64.html",
+ ],
+ )
+ )
+
+ decimal_matches = re.search(r"^decimal(.*)\((.*)\)$", datasets_dtype)
+ if decimal_matches:
+ decimal_internals_bits = decimal_matches.group(1)
+ if decimal_internals_bits == "128":
+ decimal_internals_precision_and_scale = re.search(r"^(\d+),\s*(-?\d+)$", decimal_matches.group(2))
+ if decimal_internals_precision_and_scale:
+ precision = decimal_internals_precision_and_scale.group(1)
+ scale = decimal_internals_precision_and_scale.group(2)
+ return pa.decimal128(int(precision), int(scale))
+ else:
+ raise ValueError(
+ _dtype_error_msg(
+ datasets_dtype,
+ "decimal128",
+ examples=["decimal128(10, 2)", "decimal128(4, -2)"],
+ urls=["https://arrow.apache.org/docs/python/generated/pyarrow.decimal128.html"],
+ )
+ )
+ elif decimal_internals_bits == "256":
+ decimal_internals_precision_and_scale = re.search(r"^(\d+),\s*(-?\d+)$", decimal_matches.group(2))
+ if decimal_internals_precision_and_scale:
+ precision = decimal_internals_precision_and_scale.group(1)
+ scale = decimal_internals_precision_and_scale.group(2)
+ return pa.decimal256(int(precision), int(scale))
+ else:
+ raise ValueError(
+ _dtype_error_msg(
+ datasets_dtype,
+ "decimal256",
+ examples=["decimal256(30, 2)", "decimal256(38, -4)"],
+ urls=["https://arrow.apache.org/docs/python/generated/pyarrow.decimal256.html"],
+ )
+ )
+ else:
+ raise ValueError(
+ _dtype_error_msg(
+ datasets_dtype,
+ "decimal",
+ examples=["decimal128(12, 3)", "decimal256(40, 6)"],
+ urls=[
+ "https://arrow.apache.org/docs/python/generated/pyarrow.decimal128.html",
+ "https://arrow.apache.org/docs/python/generated/pyarrow.decimal256.html",
+ ],
+ )
+ )
+
+ raise ValueError(
+ f"Neither {datasets_dtype} nor {datasets_dtype + '_'} seems to be a pyarrow data type. "
+ f"Please make sure to use a correct data type, see: "
+ f"https://arrow.apache.org/docs/python/api/datatypes.html#factory-functions"
+ )
+
+
+def _cast_to_python_objects(obj: Any, only_1d_for_numpy: bool, optimize_list_casting: bool) -> tuple[Any, bool]:
+ """
+ Cast pytorch/tensorflow/pandas objects to python numpy array/lists.
+ It works recursively.
+
+ If `optimize_list_casting` is True, to avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be casted.
+ If the first element needs to be casted, then all the elements of the list will be casted, otherwise they'll stay the same.
+ This trick allows to cast objects that contain tokenizers outputs without iterating over every single token for example.
+
+ Args:
+ obj: the object (nested struct) to cast.
+ only_1d_for_numpy (bool): whether to keep the full multi-dim tensors as multi-dim numpy arrays, or convert them to
+ nested lists of 1-dimensional numpy arrays. This can be useful to keep only 1-d arrays to instantiate Arrow arrays.
+ Indeed Arrow only support converting 1-dimensional array values.
+ optimize_list_casting (bool): whether to optimize list casting by checking the first non-null element to see if it needs to be casted
+ and if it doesn't, not checking the rest of the list elements.
+
+ Returns:
+ casted_obj: the casted object
+ has_changed (bool): True if the object has been changed, False if it is identical
+ """
+
+ if config.TF_AVAILABLE and "tensorflow" in sys.modules:
+ import tensorflow as tf
+
+ if config.TORCH_AVAILABLE and "torch" in sys.modules:
+ import torch
+
+ if config.JAX_AVAILABLE and "jax" in sys.modules:
+ import jax.numpy as jnp
+
+ if config.PIL_AVAILABLE and "PIL" in sys.modules:
+ import PIL.Image
+
+ if config.PDFPLUMBER_AVAILABLE and "pdfplumber" in sys.modules:
+ import pdfplumber
+
+ if config.NIBABEL_AVAILABLE and "nibabel" in sys.modules:
+ import nibabel as nib
+
+ if config.TORCHCODEC_AVAILABLE and "torchcodec" in sys.modules:
+ from torchcodec.decoders import AudioDecoder, VideoDecoder
+
+ if isinstance(obj, np.ndarray):
+ if obj.ndim == 0:
+ return obj[()], True
+ elif not only_1d_for_numpy or obj.ndim == 1:
+ return obj, False
+ else:
+ return (
+ [
+ _cast_to_python_objects(
+ x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
+ )[0]
+ for x in obj
+ ],
+ True,
+ )
+ elif config.TORCH_AVAILABLE and "torch" in sys.modules and isinstance(obj, torch.Tensor):
+ if obj.dtype == torch.bfloat16:
+ return _cast_to_python_objects(
+ obj.detach().to(torch.float).cpu().numpy(),
+ only_1d_for_numpy=only_1d_for_numpy,
+ optimize_list_casting=optimize_list_casting,
+ )[0], True
+ if obj.ndim == 0:
+ return obj.detach().cpu().numpy()[()], True
+ elif not only_1d_for_numpy or obj.ndim == 1:
+ return obj.detach().cpu().numpy(), True
+ else:
+ return (
+ [
+ _cast_to_python_objects(
+ x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
+ )[0]
+ for x in obj.detach().cpu().numpy()
+ ],
+ True,
+ )
+ elif config.TF_AVAILABLE and "tensorflow" in sys.modules and isinstance(obj, tf.Tensor):
+ if obj.ndim == 0:
+ return obj.numpy()[()], True
+ elif not only_1d_for_numpy or obj.ndim == 1:
+ return obj.numpy(), True
+ else:
+ return (
+ [
+ _cast_to_python_objects(
+ x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
+ )[0]
+ for x in obj.numpy()
+ ],
+ True,
+ )
+ elif config.JAX_AVAILABLE and "jax" in sys.modules and isinstance(obj, jnp.ndarray):
+ if obj.ndim == 0:
+ return np.asarray(obj)[()], True
+ elif not only_1d_for_numpy or obj.ndim == 1:
+ return np.asarray(obj), True
+ else:
+ return (
+ [
+ _cast_to_python_objects(
+ x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
+ )[0]
+ for x in np.asarray(obj)
+ ],
+ True,
+ )
+ elif config.PIL_AVAILABLE and "PIL" in sys.modules and isinstance(obj, PIL.Image.Image):
+ return encode_pil_image(obj), True
+ elif config.PDFPLUMBER_AVAILABLE and "pdfplumber" in sys.modules and isinstance(obj, pdfplumber.pdf.PDF):
+ return encode_pdfplumber_pdf(obj), True
+ elif config.NIBABEL_AVAILABLE and "nibabel" in sys.modules and isinstance(obj, nib.analyze.AnalyzeImage):
+ return encode_nibabel_image(obj, force_bytes=True), True
+ elif isinstance(obj, pd.Series):
+ return (
+ _cast_to_python_objects(
+ obj.tolist(), only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
+ )[0],
+ True,
+ )
+ elif isinstance(obj, pd.DataFrame):
+ return (
+ {
+ key: _cast_to_python_objects(
+ value, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
+ )[0]
+ for key, value in obj.to_dict("series").items()
+ },
+ True,
+ )
+ elif isinstance(obj, pd.Timestamp):
+ return obj.to_pydatetime(), True
+ elif isinstance(obj, pd.Timedelta):
+ return obj.to_pytimedelta(), True
+ elif isinstance(obj, Mapping):
+ has_changed = not isinstance(obj, dict)
+ output = {}
+ for k, v in obj.items():
+ casted_v, has_changed_v = _cast_to_python_objects(
+ v, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
+ )
+ has_changed |= has_changed_v
+ output[k] = casted_v
+ return output if has_changed else obj, has_changed
+ elif hasattr(obj, "__array__"):
+ if np.isscalar(obj):
+ return obj, False
+ else:
+ return (
+ _cast_to_python_objects(
+ obj.__array__(), only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
+ )[0],
+ True,
+ )
+ elif isinstance(obj, (list, tuple)):
+ if len(obj) > 0:
+ for first_elmt in obj:
+ if _check_non_null_non_empty_recursive(first_elmt):
+ break
+ casted_first_elmt, has_changed_first_elmt = _cast_to_python_objects(
+ first_elmt, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
+ )
+ if has_changed_first_elmt or not optimize_list_casting:
+ return (
+ [
+ _cast_to_python_objects(
+ elmt, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
+ )[0]
+ for elmt in obj
+ ],
+ True,
+ )
+ else:
+ if isinstance(obj, (list, tuple)):
+ return obj, False
+ else:
+ return list(obj), True
+ else:
+ return obj, False
+ elif config.TORCHCODEC_AVAILABLE and "torchcodec" in sys.modules and isinstance(obj, VideoDecoder):
+ v = Video()
+ return v.encode_example(obj), True
+ elif config.TORCHCODEC_AVAILABLE and "torchcodec" in sys.modules and isinstance(obj, AudioDecoder):
+ a = Audio()
+ return a.encode_example(obj), True
+ else:
+ return obj, False
+
+
+def cast_to_python_objects(obj: Any, only_1d_for_numpy=False, optimize_list_casting=True) -> Any:
+ """
+ Cast numpy/pytorch/tensorflow/pandas objects to python lists.
+ It works recursively.
+
+ If `optimize_list_casting` is True, To avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be casted.
+ If the first element needs to be casted, then all the elements of the list will be casted, otherwise they'll stay the same.
+ This trick allows to cast objects that contain tokenizers outputs without iterating over every single token for example.
+
+ Args:
+ obj: the object (nested struct) to cast
+ only_1d_for_numpy (bool, default ``False``): whether to keep the full multi-dim tensors as multi-dim numpy arrays, or convert them to
+ nested lists of 1-dimensional numpy arrays. This can be useful to keep only 1-d arrays to instantiate Arrow arrays.
+ Indeed Arrow only support converting 1-dimensional array values.
+ optimize_list_casting (bool, default ``True``): whether to optimize list casting by checking the first non-null element to see if it needs to be casted
+ and if it doesn't, not checking the rest of the list elements.
+
+ Returns:
+ casted_obj: the casted object
+ """
+ return _cast_to_python_objects(
+ obj, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting
+ )[0]
+
+
+@dataclass(repr=False)
+class Value:
+ """
+ Scalar feature value of a particular data type.
+
+ The possible dtypes of `Value` are as follows:
+ - `null`
+ - `bool`
+ - `int8`
+ - `int16`
+ - `int32`
+ - `int64`
+ - `uint8`
+ - `uint16`
+ - `uint32`
+ - `uint64`
+ - `float16`
+ - `float32` (alias float)
+ - `float64` (alias double)
+ - `time32[(s|ms)]`
+ - `time64[(us|ns)]`
+ - `timestamp[(s|ms|us|ns)]`
+ - `timestamp[(s|ms|us|ns), tz=(tzstring)]`
+ - `date32`
+ - `date64`
+ - `duration[(s|ms|us|ns)]`
+ - `decimal128(precision, scale)`
+ - `decimal256(precision, scale)`
+ - `binary`
+ - `large_binary`
+ - `binary_view`
+ - `string`
+ - `large_string`
+ - `string_view`
+
+ Args:
+ dtype (`str`):
+ Name of the data type.
+
+ Example:
+
+ ```py
+ >>> from datasets import Features
+ >>> features = Features({'stars': Value('int32')})
+ >>> features
+ {'stars': Value('int32')}
+ ```
+ """
+
+ dtype: str
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ pa_type: ClassVar[Any] = None
+ _type: str = field(default="Value", init=False, repr=False)
+
+ def __post_init__(self):
+ if self.dtype == "double": # fix inferred type
+ self.dtype = "float64"
+ if self.dtype == "float": # fix inferred type
+ self.dtype = "float32"
+ self.pa_type = string_to_arrow(self.dtype)
+
+ def __call__(self):
+ return self.pa_type
+
+ def encode_example(self, value):
+ if pa.types.is_boolean(self.pa_type):
+ return bool(value)
+ elif pa.types.is_integer(self.pa_type):
+ return int(value)
+ elif pa.types.is_floating(self.pa_type):
+ return float(value)
+ elif pa.types.is_string(self.pa_type):
+ return str(value)
+ elif pa.types.is_large_string(self.pa_type):
+ return str(value)
+ elif pa.types.is_string_view(self.pa_type):
+ return str(value)
+ else:
+ return value
+
+ def __repr__(self):
+ return f"{type(self).__name__}('{self.dtype}')"
+
+
+class _ArrayXD:
+ def __post_init__(self):
+ self.shape = tuple(self.shape)
+
+ def __call__(self):
+ pa_type = globals()[self.__class__.__name__ + "ExtensionType"](self.shape, self.dtype)
+ return pa_type
+
+ def encode_example(self, value):
+ return value
+
+
+@dataclass
+class Array2D(_ArrayXD):
+ """Create a two-dimensional array.
+
+ Args:
+ shape (`tuple`):
+ Size of each dimension.
+ dtype (`str`):
+ Name of the data type.
+
+ Example:
+
+ ```py
+ >>> from datasets import Features
+ >>> features = Features({'x': Array2D(shape=(1, 3), dtype='int32')})
+ ```
+ """
+
+ shape: tuple
+ dtype: str
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ _type: str = field(default="Array2D", init=False, repr=False)
+
+
+@dataclass
+class Array3D(_ArrayXD):
+ """Create a three-dimensional array.
+
+ Args:
+ shape (`tuple`):
+ Size of each dimension.
+ dtype (`str`):
+ Name of the data type.
+
+ Example:
+
+ ```py
+ >>> from datasets import Features
+ >>> features = Features({'x': Array3D(shape=(1, 2, 3), dtype='int32')})
+ ```
+ """
+
+ shape: tuple
+ dtype: str
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ _type: str = field(default="Array3D", init=False, repr=False)
+
+
+@dataclass
+class Array4D(_ArrayXD):
+ """Create a four-dimensional array.
+
+ Args:
+ shape (`tuple`):
+ Size of each dimension.
+ dtype (`str`):
+ Name of the data type.
+
+ Example:
+
+ ```py
+ >>> from datasets import Features
+ >>> features = Features({'x': Array4D(shape=(1, 2, 2, 3), dtype='int32')})
+ ```
+ """
+
+ shape: tuple
+ dtype: str
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ _type: str = field(default="Array4D", init=False, repr=False)
+
+
+@dataclass
+class Array5D(_ArrayXD):
+ """Create a five-dimensional array.
+
+ Args:
+ shape (`tuple`):
+ Size of each dimension.
+ dtype (`str`):
+ Name of the data type.
+
+ Example:
+
+ ```py
+ >>> from datasets import Features
+ >>> features = Features({'x': Array5D(shape=(1, 2, 2, 3, 3), dtype='int32')})
+ ```
+ """
+
+ shape: tuple
+ dtype: str
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ _type: str = field(default="Array5D", init=False, repr=False)
+
+
+class _ArrayXDExtensionType(pa.ExtensionType):
+ ndims: Optional[int] = None
+
+ def __init__(self, shape: tuple, dtype: str):
+ if self.ndims is None or self.ndims <= 1:
+ raise ValueError("You must instantiate an array type with a value for dim that is > 1")
+ if len(shape) != self.ndims:
+ raise ValueError(f"shape={shape} and ndims={self.ndims} don't match")
+ for dim in range(1, self.ndims):
+ if shape[dim] is None:
+ raise ValueError(f"Support only dynamic size on first dimension. Got: {shape}")
+ self.shape = tuple(shape)
+ self.value_type = dtype
+ self.storage_dtype = self._generate_dtype(self.value_type)
+ pa.ExtensionType.__init__(self, self.storage_dtype, f"{self.__class__.__module__}.{self.__class__.__name__}")
+
+ def __arrow_ext_serialize__(self):
+ return json.dumps((self.shape, self.value_type)).encode()
+
+ @classmethod
+ def __arrow_ext_deserialize__(cls, storage_type, serialized):
+ args = json.loads(serialized)
+ return cls(*args)
+
+ # This was added to pa.ExtensionType in pyarrow >= 13.0.0
+ def __reduce__(self):
+ return self.__arrow_ext_deserialize__, (self.storage_type, self.__arrow_ext_serialize__())
+
+ def __hash__(self):
+ return hash((self.__class__, self.shape, self.value_type))
+
+ def __arrow_ext_class__(self):
+ return ArrayExtensionArray
+
+ def _generate_dtype(self, dtype):
+ dtype = string_to_arrow(dtype)
+ for d in reversed(self.shape):
+ dtype = pa.list_(dtype)
+ # Don't specify the size of the list, since fixed length list arrays have issues
+ # being validated after slicing in pyarrow 0.17.1
+ return dtype
+
+ def to_pandas_dtype(self):
+ return PandasArrayExtensionDtype(self.value_type)
+
+
+class Array2DExtensionType(_ArrayXDExtensionType):
+ ndims = 2
+
+
+class Array3DExtensionType(_ArrayXDExtensionType):
+ ndims = 3
+
+
+class Array4DExtensionType(_ArrayXDExtensionType):
+ ndims = 4
+
+
+class Array5DExtensionType(_ArrayXDExtensionType):
+ ndims = 5
+
+
+# Register the extension types for deserialization
+pa.register_extension_type(Array2DExtensionType((1, 2), "int64"))
+pa.register_extension_type(Array3DExtensionType((1, 2, 3), "int64"))
+pa.register_extension_type(Array4DExtensionType((1, 2, 3, 4), "int64"))
+pa.register_extension_type(Array5DExtensionType((1, 2, 3, 4, 5), "int64"))
+
+
+def _is_zero_copy_only(pa_type: pa.DataType, unnest: bool = False) -> bool:
+ """
+ When converting a pyarrow array to a numpy array, we must know whether this could be done in zero-copy or not.
+ This function returns the value of the ``zero_copy_only`` parameter to pass to ``.to_numpy()``, given the type of the pyarrow array.
+
+ # zero copy is available for all primitive types except booleans and temporal types (date, time, timestamp or duration)
+ # primitive types are types for which the physical representation in arrow and in numpy
+ # https://github.com/wesm/arrow/blob/c07b9b48cf3e0bbbab493992a492ae47e5b04cad/python/pyarrow/types.pxi#L821
+ # see https://arrow.apache.org/docs/python/generated/pyarrow.Array.html#pyarrow.Array.to_numpy
+ # and https://issues.apache.org/jira/browse/ARROW-2871?jql=text%20~%20%22boolean%20to_numpy%22
+ """
+
+ def _unnest_pa_type(pa_type: pa.DataType) -> pa.DataType:
+ if pa.types.is_list(pa_type):
+ return _unnest_pa_type(pa_type.value_type)
+ return pa_type
+
+ if unnest:
+ pa_type = _unnest_pa_type(pa_type)
+ return pa.types.is_primitive(pa_type) and not (pa.types.is_boolean(pa_type) or pa.types.is_temporal(pa_type))
+
+
+class ArrayExtensionArray(pa.ExtensionArray):
+ def __array__(self):
+ zero_copy_only = _is_zero_copy_only(self.storage.type, unnest=True)
+ return self.to_numpy(zero_copy_only=zero_copy_only)
+
+ def __getitem__(self, i):
+ return self.storage[i]
+
+ def to_numpy(self, zero_copy_only=True):
+ storage: pa.ListArray = self.storage
+ null_mask = storage.is_null().to_numpy(zero_copy_only=False)
+
+ if self.type.shape[0] is not None:
+ size = 1
+ null_indices = np.arange(len(storage))[null_mask] - np.arange(np.sum(null_mask))
+
+ for i in range(self.type.ndims):
+ size *= self.type.shape[i]
+ storage = storage.flatten()
+ numpy_arr = storage.to_numpy(zero_copy_only=zero_copy_only)
+ numpy_arr = numpy_arr.reshape(len(self) - len(null_indices), *self.type.shape)
+
+ if len(null_indices):
+ numpy_arr = np.insert(numpy_arr.astype(np.float64), null_indices, np.nan, axis=0)
+
+ else:
+ shape = self.type.shape
+ ndims = self.type.ndims
+ arrays = []
+ first_dim_offsets = np.array([off.as_py() for off in storage.offsets])
+ for i, is_null in enumerate(null_mask):
+ if is_null:
+ arrays.append(np.nan)
+ else:
+ storage_el = storage[i : i + 1]
+ first_dim = first_dim_offsets[i + 1] - first_dim_offsets[i]
+ # flatten storage
+ for _ in range(ndims):
+ storage_el = storage_el.flatten()
+
+ numpy_arr = storage_el.to_numpy(zero_copy_only=zero_copy_only)
+ arrays.append(numpy_arr.reshape(first_dim, *shape[1:]))
+
+ if len(np.unique(np.diff(first_dim_offsets))) > 1:
+ # ragged
+ numpy_arr = np.empty(len(arrays), dtype=object)
+ numpy_arr[:] = arrays
+ else:
+ numpy_arr = np.array(arrays)
+
+ return numpy_arr
+
+ def to_pylist(self, maps_as_pydicts: Optional[Literal["lossy", "strict"]] = None):
+ zero_copy_only = _is_zero_copy_only(self.storage.type, unnest=True)
+ numpy_arr = self.to_numpy(zero_copy_only=zero_copy_only)
+ if self.type.shape[0] is None and numpy_arr.dtype == object:
+ return [arr.tolist() for arr in numpy_arr.tolist()]
+ else:
+ return numpy_arr.tolist()
+
+
+class PandasArrayExtensionDtype(PandasExtensionDtype):
+ _metadata = "value_type"
+
+ def __init__(self, value_type: Union["PandasArrayExtensionDtype", np.dtype]):
+ self._value_type = value_type
+
+ def __from_arrow__(self, array: Union[pa.Array, pa.ChunkedArray]):
+ if isinstance(array, pa.ChunkedArray):
+ array = array.type.wrap_array(pa.concat_arrays([chunk.storage for chunk in array.chunks]))
+ zero_copy_only = _is_zero_copy_only(array.storage.type, unnest=True)
+ numpy_arr = array.to_numpy(zero_copy_only=zero_copy_only)
+ return PandasArrayExtensionArray(numpy_arr)
+
+ @classmethod
+ def construct_array_type(cls):
+ return PandasArrayExtensionArray
+
+ @property
+ def type(self) -> type:
+ return np.ndarray
+
+ @property
+ def kind(self) -> str:
+ return "O"
+
+ @property
+ def name(self) -> str:
+ return f"array[{self.value_type}]"
+
+ @property
+ def value_type(self) -> np.dtype:
+ return self._value_type
+
+
+class PandasArrayExtensionArray(PandasExtensionArray):
+ def __init__(self, data: np.ndarray, copy: bool = False):
+ self._data = data if not copy else np.array(data)
+ self._dtype = PandasArrayExtensionDtype(data.dtype)
+
+ def __array__(self, dtype=None):
+ """
+ Convert to NumPy Array.
+ Note that Pandas expects a 1D array when dtype is set to object.
+ But for other dtypes, the returned shape is the same as the one of ``data``.
+
+ More info about pandas 1D requirement for PandasExtensionArray here:
+ https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.api.extensions.ExtensionArray.html#pandas.api.extensions.ExtensionArray
+
+ """
+ if dtype == np.dtype(object):
+ out = np.empty(len(self._data), dtype=object)
+ for i in range(len(self._data)):
+ out[i] = self._data[i]
+ return out
+ if dtype is None:
+ return self._data
+ else:
+ return self._data.astype(dtype)
+
+ def copy(self, deep: bool = False) -> "PandasArrayExtensionArray":
+ return PandasArrayExtensionArray(self._data, copy=True)
+
+ @classmethod
+ def _from_sequence(
+ cls, scalars, dtype: Optional[PandasArrayExtensionDtype] = None, copy: bool = False
+ ) -> "PandasArrayExtensionArray":
+ if len(scalars) > 1 and all(
+ isinstance(x, np.ndarray) and x.shape == scalars[0].shape and x.dtype == scalars[0].dtype for x in scalars
+ ):
+ data = np.array(scalars, dtype=dtype if dtype is None else dtype.value_type, copy=copy)
+ else:
+ data = np.empty(len(scalars), dtype=object)
+ data[:] = scalars
+ return cls(data, copy=copy)
+
+ @classmethod
+ def _concat_same_type(cls, to_concat: Sequence_["PandasArrayExtensionArray"]) -> "PandasArrayExtensionArray":
+ if len(to_concat) > 1 and all(
+ va._data.shape == to_concat[0]._data.shape and va._data.dtype == to_concat[0]._data.dtype
+ for va in to_concat
+ ):
+ data = np.vstack([va._data for va in to_concat])
+ else:
+ data = np.empty(len(to_concat), dtype=object)
+ data[:] = [va._data for va in to_concat]
+ return cls(data, copy=False)
+
+ @property
+ def dtype(self) -> PandasArrayExtensionDtype:
+ return self._dtype
+
+ @property
+ def nbytes(self) -> int:
+ return self._data.nbytes
+
+ def isna(self) -> np.ndarray:
+ return np.array([pd.isna(arr).any() for arr in self._data])
+
+ def __setitem__(self, key: Union[int, slice, np.ndarray], value: Any) -> None:
+ raise NotImplementedError()
+
+ def __getitem__(self, item: Union[int, slice, np.ndarray]) -> Union[np.ndarray, "PandasArrayExtensionArray"]:
+ if isinstance(item, int):
+ return self._data[item]
+ return PandasArrayExtensionArray(self._data[item], copy=False)
+
+ def take(
+ self, indices: Sequence_[int], allow_fill: bool = False, fill_value: bool = None
+ ) -> "PandasArrayExtensionArray":
+ indices: np.ndarray = np.asarray(indices, dtype=int)
+ if allow_fill:
+ fill_value = (
+ self.dtype.na_value if fill_value is None else np.asarray(fill_value, dtype=self.dtype.value_type)
+ )
+ mask = indices == -1
+ if (indices < -1).any():
+ raise ValueError("Invalid value in `indices`, must be all >= -1 for `allow_fill` is True")
+ elif len(self) > 0:
+ pass
+ elif not np.all(mask):
+ raise IndexError("Invalid take for empty PandasArrayExtensionArray, must be all -1.")
+ else:
+ data = np.array([fill_value] * len(indices), dtype=self.dtype.value_type)
+ return PandasArrayExtensionArray(data, copy=False)
+ took = self._data.take(indices, axis=0)
+ if allow_fill and mask.any():
+ took[mask] = [fill_value] * np.sum(mask)
+ return PandasArrayExtensionArray(took, copy=False)
+
+ def __len__(self) -> int:
+ return len(self._data)
+
+ def __eq__(self, other) -> np.ndarray:
+ if not isinstance(other, PandasArrayExtensionArray):
+ raise NotImplementedError(f"Invalid type to compare to: {type(other)}")
+ return (self._data == other._data).all()
+
+
+def pandas_types_mapper(dtype):
+ if isinstance(dtype, _ArrayXDExtensionType):
+ return PandasArrayExtensionDtype(dtype.value_type)
+
+
+@dataclass
+class ClassLabel:
+ """Feature type for integer class labels.
+
+ There are 3 ways to define a `ClassLabel`, which correspond to the 3 arguments:
+
+ * `num_classes`: Create 0 to (num_classes-1) labels.
+ * `names`: List of label strings.
+ * `names_file`: File containing the list of labels.
+
+ Under the hood the labels are stored as integers.
+ You can use -1 to represent unknown/missing labels.
+
+ Args:
+ num_classes (`int`, *optional*):
+ Number of classes. All labels must be < `num_classes`.
+ names (`list` of `str`, *optional*):
+ String names for the integer classes.
+ The order in which the names are provided is kept.
+ names_file (`str`, *optional*):
+ Path to a file with names for the integer classes, one per line.
+
+ Example:
+
+ ```py
+ >>> from datasets import Features, ClassLabel
+ >>> features = Features({'label': ClassLabel(num_classes=3, names=['bad', 'ok', 'good'])})
+ >>> features
+ {'label': ClassLabel(names=['bad', 'ok', 'good'])}
+ ```
+ """
+
+ num_classes: InitVar[Optional[int]] = None # Pseudo-field: ignored by asdict/fields when converting to/from dict
+ names: list[str] = None
+ names_file: InitVar[Optional[str]] = None # Pseudo-field: ignored by asdict/fields when converting to/from dict
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ dtype: ClassVar[str] = "int64"
+ pa_type: ClassVar[Any] = pa.int64()
+ _str2int: ClassVar[dict[str, int]] = None
+ _int2str: ClassVar[dict[int, int]] = None
+ _type: str = field(default="ClassLabel", init=False, repr=False)
+
+ def __post_init__(self, num_classes, names_file):
+ self.num_classes = num_classes
+ self.names_file = names_file
+ if self.names_file is not None and self.names is not None:
+ raise ValueError("Please provide either names or names_file but not both.")
+ # Set self.names
+ if self.names is None:
+ if self.names_file is not None:
+ self.names = self._load_names_from_file(self.names_file)
+ elif self.num_classes is not None:
+ self.names = [str(i) for i in range(self.num_classes)]
+ else:
+ raise ValueError("Please provide either num_classes, names or names_file.")
+ elif not isinstance(self.names, SequenceABC):
+ raise TypeError(f"Please provide names as a list, is {type(self.names)}")
+ # Set self.num_classes
+ if self.num_classes is None:
+ self.num_classes = len(self.names)
+ elif self.num_classes != len(self.names):
+ raise ValueError(
+ "ClassLabel number of names do not match the defined num_classes. "
+ f"Got {len(self.names)} names VS {self.num_classes} num_classes"
+ )
+ # Prepare mappings
+ self._int2str = [str(name) for name in self.names]
+ self._str2int = {name: i for i, name in enumerate(self._int2str)}
+ if len(self._int2str) != len(self._str2int):
+ raise ValueError("Some label names are duplicated. Each label name should be unique.")
+
+ def __call__(self):
+ return self.pa_type
+
+ def str2int(self, values: Union[str, Iterable]) -> Union[int, Iterable]:
+ """Conversion class name `string` => `integer`.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
+ >>> ds.features["label"].str2int('neg')
+ 0
+ ```
+ """
+ if not isinstance(values, str) and not isinstance(values, Iterable):
+ raise ValueError(
+ f"Values {values} should be a string or an Iterable (list, numpy array, pytorch, tensorflow tensors)"
+ )
+ return_list = True
+ if isinstance(values, str):
+ values = [values]
+ return_list = False
+
+ output = [self._strval2int(value) for value in values]
+ return output if return_list else output[0]
+
+ def _strval2int(self, value: str) -> int:
+ failed_parse = False
+ value = str(value)
+ # first attempt - raw string value
+ int_value = self._str2int.get(value)
+ if int_value is None:
+ # second attempt - strip whitespace
+ int_value = self._str2int.get(value.strip())
+ if int_value is None:
+ # third attempt - convert str to int
+ try:
+ int_value = int(value)
+ except ValueError:
+ failed_parse = True
+ else:
+ if int_value < -1 or int_value >= self.num_classes:
+ failed_parse = True
+ if failed_parse:
+ raise ValueError(f"Invalid string class label {value}")
+ return int_value
+
+ def int2str(self, values: Union[int, Iterable]) -> Union[str, Iterable]:
+ """Conversion `integer` => class name `string`.
+
+ Regarding unknown/missing labels: passing negative integers raises `ValueError`.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
+ >>> ds.features["label"].int2str(0)
+ 'neg'
+ ```
+ """
+ if not isinstance(values, int) and not isinstance(values, Iterable):
+ raise ValueError(
+ f"Values {values} should be an integer or an Iterable (list, numpy array, pytorch, tensorflow tensors)"
+ )
+ return_list = True
+ if isinstance(values, int):
+ values = [values]
+ return_list = False
+
+ for v in values:
+ if not 0 <= v < self.num_classes:
+ raise ValueError(f"Invalid integer class label {v:d}")
+
+ output = [self._int2str[int(v)] for v in values]
+ return output if return_list else output[0]
+
+ def encode_example(self, example_data):
+ if self.num_classes is None:
+ raise ValueError(
+ "Trying to use ClassLabel feature with undefined number of class. "
+ "Please set ClassLabel.names or num_classes."
+ )
+
+ # If a string is given, convert to associated integer
+ if isinstance(example_data, str):
+ example_data = self.str2int(example_data)
+
+ # Allowing -1 to mean no label.
+ if not -1 <= example_data < self.num_classes:
+ raise ValueError(f"Class label {example_data:d} greater than configured num_classes {self.num_classes}")
+ return example_data
+
+ def cast_storage(self, storage: Union[pa.StringArray, pa.IntegerArray]) -> pa.Int64Array:
+ """Cast an Arrow array to the `ClassLabel` arrow storage type.
+ The Arrow types that can be converted to the `ClassLabel` pyarrow storage type are:
+
+ - `pa.string()`
+ - `pa.int()`
+
+ Args:
+ storage (`Union[pa.StringArray, pa.IntegerArray]`):
+ PyArrow array to cast.
+
+ Returns:
+ `pa.Int64Array`: Array in the `ClassLabel` arrow storage type.
+ """
+ if isinstance(storage, pa.IntegerArray) and len(storage) > 0:
+ min_max = pc.min_max(storage).as_py()
+ if min_max["max"] is not None and min_max["max"] >= self.num_classes:
+ raise ValueError(
+ f"Class label {min_max['max']} greater than configured num_classes {self.num_classes}"
+ )
+ elif isinstance(storage, pa.StringArray):
+ storage = pa.array(
+ [self._strval2int(label) if label is not None else None for label in storage.to_pylist()]
+ )
+ return array_cast(storage, self.pa_type)
+
+ @staticmethod
+ def _load_names_from_file(names_filepath):
+ with open(names_filepath, encoding="utf-8") as f:
+ return [name.strip() for name in f.read().split("\n") if name.strip()] # Filter empty names
+
+
+@dataclass
+class Json:
+ """Feature type for JSON objects.
+
+ Under the hood the objects are stored as JSON-encoded strings.
+
+ Example:
+
+ ```py
+ >>> from datasets import Features, Json
+ >>> features = Features({'json': Json()})
+ >>> features
+ {'json': Json()}
+ ```
+
+ ```py
+ >>> from datasets import Dataset, Features, Json, List
+ >>> features = Features({"a": List(Json())})
+ >>> ds = Dataset.from_dict({"a": [[{"b": 0}, {"c": 0}]]}, features=features)
+ >>> # OR
+ >>> ds = Dataset.from_dict({"a": [[{"b": 0}, {"c": 0}]]}, on_mixed_types="use_json")
+ >>> ds.features
+ {'a': List(Json())}
+ >>> ds[0]
+ {'a': [{'b': 0}, {'c': 0}]}
+ >>> def f(x):
+ ... for y in x["a"]:
+ ... y["d"] = "foo"
+ ... return x
+ >>> ds = ds.map(f)
+ >>> ds.features
+ >>> ds[0]
+ {'a': [{'b': 0, 'd': 'foo'}, {'c': 0, 'd': 'foo'}]}
+ ```
+ """
+
+ decode: bool = True
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ pa_type: ClassVar[Any] = pa.json_()
+ _type: str = field(default="Json", init=False, repr=False)
+
+ def __call__(self):
+ return self.pa_type
+
+ def encode_example(self, example_data):
+ if example_data is None:
+ return None
+ if not isinstance(example_data, str):
+ example_data = ujson_dumps(example_data)
+ else:
+ try:
+ ujson_loads(example_data)
+ except Exception:
+ example_data = ujson_dumps(example_data)
+ return example_data
+
+ def decode_example(self, example_data, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None):
+ if not self.decode:
+ raise RuntimeError("Decoding is disabled for this feature. Please use Json(decode=True) instead.")
+ if example_data is None:
+ return None
+ return ujson_loads(example_data)
+
+ def cast_storage(self, storage: Union[pa.Array]) -> pa.JsonArray:
+ """Cast an Arrow array to the `Json` arrow storage type.
+
+ Args:
+ storage (`Union[pa.StringArray, pa.IntegerArray]`):
+ PyArrow array to cast.
+
+ Returns:
+ `pa.JsonArray`: Array in the `Json` arrow storage type.
+ """
+ if isinstance(storage, pa.JsonArray):
+ return storage
+ elif isinstance(storage, (pa.StringArray)):
+ items = storage[:5].to_pylist()
+ try:
+ for item in items:
+ if item is not None:
+ ujson_loads(item)
+ except Exception:
+ storage = pa.array(
+ [ujson_dumps(x) if x is not None else None for x in storage.to_pylist()], pa.json_()
+ )
+ else:
+ storage = pa.array([ujson_dumps(x) if x is not None else None for x in storage.to_pylist()], pa.json_())
+ return array_cast(storage, self.pa_type)
+
+
+class Sequence:
+ """
+ A `Sequence` is a utility that automatically converts internal dictionary feature into a dictionary of
+ lists. This behavior is implemented to have a compatibility layer with the TensorFlow Datasets library but may be
+ un-wanted in some cases. If you don't want this behavior, you can use a [`List`] or a [`LargeList`]
+ instead of the [`Sequence`].
+
+ Args:
+ feature ([`FeatureType`]):
+ Child feature data type of each item within the large list.
+ length (optional `int`, default to -1):
+ Length of the list if it is fixed.
+ Defaults to -1 which means an arbitrary length.
+
+ Returns:
+ [`List`] of the specified feature, except `dict` of sub-features
+ which are converted to `dict` of lists of sub-features for compatibility with TFDS.
+
+ """
+
+ def __new__(cls, feature=None, length=-1, **kwargs):
+ # useful to still get isinstance(Sequence(Value("int64")), Sequence)
+ if (
+ cls is Sequence
+ and isinstance(feature, dict)
+ and any(not isinstance(subfeature, List) for subfeature in feature.values())
+ ):
+ out = {key: List(value, length=length, **kwargs) for key, value in feature.items()}
+ else:
+ out = super().__new__(List)
+ return out
+
+
+@dataclass(repr=False)
+class List(Sequence):
+ """Feature type for large list data composed of child feature data type.
+
+ It is backed by `pyarrow.ListType`, which uses 32-bit offsets or a fixed length.
+
+ Args:
+ feature ([`FeatureType`]):
+ Child feature data type of each item within the large list.
+ length (optional `int`, default to -1):
+ Length of the list if it is fixed.
+ Defaults to -1 which means an arbitrary length.
+ """
+
+ feature: Any
+ length: int = -1
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ pa_type: ClassVar[Any] = None
+ _type: str = field(default="List", init=False, repr=False)
+
+ def __repr__(self):
+ if self.length != -1:
+ return f"{type(self).__name__}({self.feature}, length={self.length})"
+ else:
+ return f"{type(self).__name__}({self.feature})"
+
+
+@dataclass(repr=False)
+class LargeList:
+ """Feature type for large list data composed of child feature data type.
+
+ It is backed by `pyarrow.LargeListType`, which is like `pyarrow.ListType` but with 64-bit rather than 32-bit offsets.
+
+ Args:
+ feature ([`FeatureType`]):
+ Child feature data type of each item within the large list.
+ """
+
+ feature: Any
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ pa_type: ClassVar[Any] = None
+ _type: str = field(default="LargeList", init=False, repr=False)
+
+ def __repr__(self):
+ return f"{type(self).__name__}({self.feature})"
+
+
+FeatureType = Union[
+ dict,
+ list,
+ tuple,
+ Value,
+ ClassLabel,
+ Translation,
+ TranslationVariableLanguages,
+ LargeList,
+ List,
+ Array2D,
+ Array3D,
+ Array4D,
+ Array5D,
+ Audio,
+ Image,
+ Mesh,
+ Video,
+ Pdf,
+ Nifti,
+]
+
+
+def _check_non_null_non_empty_recursive(obj, schema: Optional[FeatureType] = None) -> bool:
+ """
+ Check if the object is not None.
+ If the object is a list or a tuple, recursively check the first element of the sequence and stop if at any point the first element is not a sequence or is an empty sequence.
+ """
+ if obj is None:
+ return False
+ elif isinstance(obj, (list, tuple)) and (schema is None or isinstance(schema, (list, tuple, LargeList, List))):
+ if len(obj) > 0:
+ if schema is None:
+ pass
+ elif isinstance(schema, (list, tuple)):
+ schema = schema[0]
+ else:
+ schema = schema.feature
+ return _check_non_null_non_empty_recursive(obj[0], schema)
+ else:
+ return False
+ else:
+ return True
+
+
+def get_nested_type(schema: FeatureType) -> pa.DataType:
+ """
+ get_nested_type() converts a datasets.FeatureType into a pyarrow.DataType, and acts as the inverse of
+ generate_from_arrow_type().
+
+ It performs double-duty as the implementation of Features.type and handles the conversion of
+ datasets.Feature->pa.struct
+ """
+ # Nested structures: we allow dict, list/tuples, sequences
+ if isinstance(schema, Features):
+ return pa.struct(
+ {key: get_nested_type(schema[key]) for key in schema}
+ ) # Features is subclass of dict, and dict order is deterministic since Python 3.6
+ elif isinstance(schema, dict):
+ return pa.struct(
+ {key: get_nested_type(schema[key]) for key in schema}
+ ) # however don't sort on struct types since the order matters
+ elif isinstance(schema, (list, tuple)):
+ if len(schema) != 1:
+ raise ValueError("When defining list feature, you should just provide one example of the inner type")
+ value_type = get_nested_type(schema[0])
+ return pa.list_(value_type)
+ elif isinstance(schema, LargeList):
+ value_type = get_nested_type(schema.feature)
+ return pa.large_list(value_type)
+ elif isinstance(schema, List):
+ value_type = get_nested_type(schema.feature)
+ return pa.list_(value_type, schema.length)
+
+ # Other objects are callable which returns their data type (ClassLabel, Array2D, Translation, Arrow datatype creation methods)
+ return schema()
+
+
+def encode_nested_example(schema, obj, level=0):
+ """Encode a nested example.
+ This is used since some features (in particular ClassLabel) have some logic during encoding.
+
+ To avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be encoded.
+ If the first element needs to be encoded, then all the elements of the list will be encoded, otherwise they'll stay the same.
+ """
+ # Nested structures: we allow dict, list/tuples, sequences
+ if isinstance(schema, dict):
+ if level == 0 and obj is None:
+ raise ValueError("Got None but expected a dictionary instead")
+ return (
+ {k: encode_nested_example(schema[k], obj.get(k), level=level + 1) for k in schema}
+ if obj is not None
+ else None
+ )
+ elif isinstance(schema, (LargeList, List)):
+ if obj is None:
+ return None
+ else:
+ if len(obj) > 0:
+ sub_schema = schema.feature
+ for first_elmt in obj:
+ if _check_non_null_non_empty_recursive(first_elmt, sub_schema):
+ break
+ try:
+ changed = bool(encode_nested_example(sub_schema, first_elmt, level=level + 1) != first_elmt)
+ except ValueError: # can happen when comparing arrays
+ changed = False
+ if changed:
+ return [encode_nested_example(sub_schema, o, level=level + 1) for o in obj]
+ return list(obj)
+ # Object with special encoding:
+ # ClassLabel will convert from string to int, TranslationVariableLanguages does some checks
+ elif hasattr(schema, "encode_example"):
+ return schema.encode_example(obj) if obj is not None else None
+ # Other object should be directly convertible to a native Arrow type (like Translation and Translation)
+ return obj
+
+
+def decode_nested_example(schema, obj, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None):
+ """Decode a nested example.
+ This is used since some features (in particular Audio and Image) have some logic during decoding.
+
+ To avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be decoded.
+ If the first element needs to be decoded, then all the elements of the list will be decoded, otherwise they'll stay the same.
+ """
+ # Nested structures: we allow dict, list/tuples, sequences
+ if isinstance(schema, dict):
+ return (
+ {k: decode_nested_example(sub_schema, sub_obj) for k, (sub_schema, sub_obj) in zip_dict(schema, obj)}
+ if obj is not None
+ else None
+ )
+ elif isinstance(schema, (list, tuple)):
+ sub_schema = schema[0]
+ if obj is None:
+ return None
+ else:
+ if len(obj) > 0:
+ for first_elmt in obj:
+ if _check_non_null_non_empty_recursive(first_elmt, sub_schema):
+ break
+ if decode_nested_example(sub_schema, first_elmt) != first_elmt:
+ return [decode_nested_example(sub_schema, o) for o in obj]
+ return list(obj)
+ elif isinstance(schema, (LargeList, List)):
+ if obj is None:
+ return None
+ else:
+ sub_schema = schema.feature
+ if isinstance(sub_schema, dict):
+ return [decode_nested_example(sub_schema, o) for o in obj]
+ if len(obj) > 0:
+ for first_elmt in obj:
+ if _check_non_null_non_empty_recursive(first_elmt, sub_schema):
+ break
+ if decode_nested_example(sub_schema, first_elmt) != first_elmt:
+ return [decode_nested_example(sub_schema, o) for o in obj]
+ return list(obj)
+ # Object with special decoding:
+ elif hasattr(schema, "decode_example") and getattr(schema, "decode", True):
+ # we pass the token to read and decode files from private repositories in streaming mode
+ return schema.decode_example(obj, token_per_repo_id=token_per_repo_id) if obj is not None else None
+ return obj
+
+
+_FEATURE_TYPES: dict[str, FeatureType] = {
+ Value.__name__: Value,
+ ClassLabel.__name__: ClassLabel,
+ Translation.__name__: Translation,
+ TranslationVariableLanguages.__name__: TranslationVariableLanguages,
+ LargeList.__name__: LargeList,
+ List.__name__: List,
+ Array2D.__name__: Array2D,
+ Array3D.__name__: Array3D,
+ Array4D.__name__: Array4D,
+ Array5D.__name__: Array5D,
+ Audio.__name__: Audio,
+ Image.__name__: Image,
+ Mesh.__name__: Mesh,
+ Video.__name__: Video,
+ Pdf.__name__: Pdf,
+ Nifti.__name__: Nifti,
+ Json.__name__: Json,
+}
+
+
+@experimental
+def register_feature(
+ feature_cls: type,
+ feature_type: str,
+):
+ """
+ Register a Feature object using a name and class.
+ This function must be used on a Feature class.
+ """
+ if feature_type in _FEATURE_TYPES:
+ logger.warning(
+ f"Overwriting feature type '{feature_type}' ({_FEATURE_TYPES[feature_type].__name__} -> {feature_cls.__name__})"
+ )
+ _FEATURE_TYPES[feature_type] = feature_cls
+
+
+def generate_from_dict(obj: Any):
+ """Regenerate the nested feature object from a deserialized dict.
+ We use the '_type' fields to get the dataclass name to load.
+
+ generate_from_dict is the recursive helper for Features.from_dict, and allows for a convenient constructor syntax
+ to define features from deserialized JSON dictionaries. This function is used in particular when deserializing
+ a :class:`DatasetInfo` that was dumped to a JSON object. This acts as an analogue to
+ :meth:`Features.from_arrow_schema` and handles the recursive field-by-field instantiation, but doesn't require any
+ mapping to/from pyarrow, except for the fact that it takes advantage of the mapping of pyarrow primitive dtypes
+ that :class:`Value` automatically performs.
+ """
+ # Nested structures: we allow dict, list/tuples, sequences
+ if isinstance(obj, list):
+ return [generate_from_dict(value) for value in obj]
+ # Otherwise we have a dict or a dataclass
+ if "_type" not in obj or isinstance(obj["_type"], dict):
+ return {key: generate_from_dict(value) for key, value in obj.items()}
+ obj = dict(obj)
+ _type = obj.pop("_type")
+ class_type = _FEATURE_TYPES.get(_type, None) or globals().get(_type, None)
+
+ if class_type is None:
+ raise ValueError(f"Feature type '{_type}' not found. Available feature types: {list(_FEATURE_TYPES.keys())}")
+
+ if class_type == LargeList:
+ feature = obj.pop("feature")
+ return LargeList(generate_from_dict(feature), **obj)
+ if class_type == List:
+ feature = obj.pop("feature")
+ return List(generate_from_dict(feature), **obj)
+ if class_type == Sequence: # backward compatibility, this translates to a List or a dict
+ feature = obj.pop("feature")
+ return Sequence(feature=generate_from_dict(feature), **obj)
+
+ field_names = {f.name for f in fields(class_type)}
+ return class_type(**{k: v for k, v in obj.items() if k in field_names})
+
+
+def generate_from_arrow_type(pa_type: pa.DataType) -> FeatureType:
+ """
+ generate_from_arrow_type accepts an arrow DataType and returns a datasets FeatureType to be used as the type for
+ a single field.
+
+ This is the high-level arrow->datasets type conversion and is inverted by get_nested_type().
+
+ This operates at the individual *field* level, whereas Features.from_arrow_schema() operates at the
+ full schema level and holds the methods that represent the bijection from Features<->pyarrow.Schema
+ """
+ if isinstance(pa_type, pa.StructType):
+ return {field.name: generate_from_arrow_type(field.type) for field in pa_type}
+ elif isinstance(pa_type, pa.FixedSizeListType):
+ return List(generate_from_arrow_type(pa_type.value_type), length=pa_type.list_size)
+ elif isinstance(pa_type, pa.ListType):
+ return List(generate_from_arrow_type(pa_type.value_type))
+ elif isinstance(pa_type, pa.LargeListType):
+ return LargeList(generate_from_arrow_type(pa_type.value_type))
+ elif isinstance(pa_type, _ArrayXDExtensionType):
+ array_feature = [None, None, Array2D, Array3D, Array4D, Array5D][pa_type.ndims]
+ return array_feature(shape=pa_type.shape, dtype=pa_type.value_type)
+ elif isinstance(pa_type, pa.JsonType):
+ return Json()
+ elif isinstance(pa_type, pa.DataType):
+ return Value(dtype=_arrow_to_datasets_dtype(pa_type))
+ else:
+ raise ValueError(f"Cannot convert {pa_type} to a Feature type.")
+
+
+def numpy_to_pyarrow_listarray(arr: np.ndarray, type: pa.DataType = None) -> pa.ListArray:
+ """Build a PyArrow ListArray from a multidimensional NumPy array"""
+ arr = np.array(arr)
+ values = pa.array(arr.flatten(), type=type)
+ for i in range(arr.ndim - 1):
+ n_offsets = reduce(mul, arr.shape[: arr.ndim - i - 1], 1)
+ step_offsets = arr.shape[arr.ndim - i - 1]
+ offsets = pa.array(np.arange(n_offsets + 1) * step_offsets, type=pa.int32())
+ values = pa.ListArray.from_arrays(offsets, values)
+ return values
+
+
+def list_of_pa_arrays_to_pyarrow_listarray(l_arr: list[Optional[pa.Array]]) -> pa.ListArray:
+ null_mask = np.array([arr is None for arr in l_arr])
+ null_indices = np.arange(len(null_mask))[null_mask] - np.arange(np.sum(null_mask))
+ l_arr = [arr for arr in l_arr if arr is not None]
+ offsets = np.cumsum(
+ [0] + [len(arr) for arr in l_arr], dtype=object
+ ) # convert to dtype object to allow None insertion
+ offsets = np.insert(offsets, null_indices, None)
+ offsets = pa.array(offsets, type=pa.int32())
+ values = pa.concat_arrays(l_arr)
+ return pa.ListArray.from_arrays(offsets, values)
+
+
+def list_of_np_array_to_pyarrow_listarray(l_arr: list[np.ndarray], type: pa.DataType = None) -> pa.ListArray:
+ """Build a PyArrow ListArray from a possibly nested list of NumPy arrays"""
+ if len(l_arr) > 0:
+ return list_of_pa_arrays_to_pyarrow_listarray(
+ [numpy_to_pyarrow_listarray(arr, type=type) if arr is not None else None for arr in l_arr]
+ )
+ else:
+ return pa.array([], type=type)
+
+
+def contains_any_np_array(data: Any):
+ """Return `True` if data is a NumPy ndarray or (recursively) if first non-null value in list is a NumPy ndarray.
+
+ Args:
+ data (Any): Data.
+
+ Returns:
+ bool
+ """
+ if isinstance(data, np.ndarray):
+ return True
+ elif isinstance(data, list):
+ return contains_any_np_array(first_non_null_value(data)[1])
+ else:
+ return False
+
+
+def any_np_array_to_pyarrow_listarray(data: Union[np.ndarray, list], type: pa.DataType = None) -> pa.ListArray:
+ """Convert to PyArrow ListArray either a NumPy ndarray or (recursively) a list that may contain any NumPy ndarray.
+
+ Args:
+ data (Union[np.ndarray, List]): Data.
+ type (pa.DataType): Explicit PyArrow DataType passed to coerce the ListArray data type.
+
+ Returns:
+ pa.ListArray
+ """
+ if isinstance(data, np.ndarray):
+ return numpy_to_pyarrow_listarray(data, type=type)
+ elif isinstance(data, list):
+ return list_of_pa_arrays_to_pyarrow_listarray([any_np_array_to_pyarrow_listarray(i, type=type) for i in data])
+
+
+def to_pyarrow_listarray(data: Any, pa_type: _ArrayXDExtensionType) -> pa.Array:
+ """Convert to PyArrow ListArray.
+
+ Args:
+ data (Any): List, iterable, np.ndarray or pd.Series.
+ pa_type (_ArrayXDExtensionType): Any of the ArrayNDExtensionType.
+
+ Returns:
+ pyarrow.Array
+ """
+ if contains_any_np_array(data):
+ return any_np_array_to_pyarrow_listarray(data, type=pa_type.value_type)
+ else:
+ return pa.array(data, pa_type.storage_dtype)
+
+
+def _visit(feature: FeatureType, func: Callable[[FeatureType], Optional[FeatureType]]) -> FeatureType:
+ """Visit a (possibly nested) feature.
+
+ Args:
+ feature (FeatureType): the feature type to be checked
+ Returns:
+ visited feature (FeatureType)
+ """
+ if isinstance(feature, Features):
+ out = func(Features({k: _visit(f, func) for k, f in feature.items()}))
+ elif isinstance(feature, dict):
+ out = func({k: _visit(f, func) for k, f in feature.items()})
+ elif isinstance(feature, LargeList):
+ out = func(LargeList(_visit(feature.feature, func)))
+ elif isinstance(feature, List):
+ out = func(List(_visit(feature.feature, func), length=feature.length))
+ else:
+ out = func(feature)
+ return feature if out is None else out
+
+
+_VisitPath = list[Union[str, Literal[0]]]
+
+
+def _visit_with_path(
+ feature: FeatureType, func: Callable[[FeatureType, _VisitPath], Optional[FeatureType]], visit_path: _VisitPath = []
+) -> FeatureType:
+ """Visit a (possibly nested) feature with its path in the Feature object.
+
+ A path in a nested feature object is the list of keys that need to be
+ sequentially accessed to get to the sub-feature.
+
+ For example:
+ - ["foo"] corresponds to the column "foo"
+ - ["foo", 0] corresponds to the sub-feature of the lists in "foo"
+ - ["foo", "bar"] corresponds to the sub-feature of the dicts in "foo" with key "bar"
+
+ Args:
+ feature (`FeatureType`): the feature type to be checked.
+
+ Returns:
+ `FeatureType`: the visited feature.
+ """
+ if isinstance(feature, Features):
+ out = func(Features({k: _visit_with_path(f, func, visit_path + [k]) for k, f in feature.items()}), visit_path)
+ elif isinstance(feature, dict):
+ out = func({k: _visit_with_path(f, func, visit_path + [k]) for k, f in feature.items()}, visit_path)
+ elif isinstance(feature, List):
+ out = func(List(_visit_with_path(feature.feature, func, visit_path + [0]), length=feature.length), visit_path)
+ elif isinstance(feature, LargeList):
+ out = func(LargeList(_visit_with_path(feature.feature, func, visit_path + [0])), visit_path)
+ else:
+ out = func(feature, visit_path)
+ return feature if out is None else out
+
+
+def require_decoding(feature: FeatureType, ignore_decode_attribute: bool = False) -> bool:
+ """Check if a (possibly nested) feature requires decoding.
+
+ Args:
+ feature (FeatureType): the feature type to be checked
+ ignore_decode_attribute (:obj:`bool`, default ``False``): Whether to ignore the current value
+ of the `decode` attribute of the decodable feature types.
+ Returns:
+ :obj:`bool`
+ """
+ if isinstance(feature, dict):
+ return any(require_decoding(f) for f in feature.values())
+ elif isinstance(feature, (list, tuple)):
+ return require_decoding(feature[0])
+ elif isinstance(feature, LargeList):
+ return require_decoding(feature.feature)
+ elif isinstance(feature, List):
+ return require_decoding(feature.feature)
+ else:
+ return hasattr(feature, "decode_example") and (
+ getattr(feature, "decode", True) if not ignore_decode_attribute else True
+ )
+
+
+def require_storage_cast(feature: FeatureType) -> bool:
+ """Check if a (possibly nested) feature requires storage casting.
+
+ Args:
+ feature (FeatureType): the feature type to be checked
+ Returns:
+ :obj:`bool`
+ """
+ if isinstance(feature, dict):
+ return any(require_storage_cast(f) for f in feature.values())
+ elif isinstance(feature, LargeList):
+ return require_storage_cast(feature.feature)
+ elif isinstance(feature, List):
+ return require_storage_cast(feature.feature)
+ else:
+ return hasattr(feature, "cast_storage")
+
+
+def require_storage_embed(feature: FeatureType) -> bool:
+ """Check if a (possibly nested) feature requires embedding data into storage.
+
+ Args:
+ feature (FeatureType): the feature type to be checked
+ Returns:
+ :obj:`bool`
+ """
+ if isinstance(feature, dict):
+ return any(require_storage_cast(f) for f in feature.values())
+ elif isinstance(feature, LargeList):
+ return require_storage_cast(feature.feature)
+ elif isinstance(feature, List):
+ return require_storage_cast(feature.feature)
+ else:
+ return hasattr(feature, "embed_storage")
+
+
+def keep_features_dicts_synced(func):
+ """
+ Wrapper to keep the secondary dictionary, which tracks whether keys are decodable, of the :class:`datasets.Features` object
+ in sync with the main dictionary.
+ """
+
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ if args:
+ self: "Features" = args[0]
+ args = args[1:]
+ else:
+ self: "Features" = kwargs.pop("self")
+ out = func(self, *args, **kwargs)
+ assert hasattr(self, "_column_requires_decoding")
+ self._column_requires_decoding = {col: require_decoding(feature) for col, feature in self.items()}
+ return out
+
+ wrapper._decorator_name_ = "_keep_dicts_synced"
+ return wrapper
+
+
+class Features(dict):
+ """A special dictionary that defines the internal structure of a dataset.
+
+ Instantiated with a dictionary of type `dict[str, FieldType]`, where keys are the desired column names,
+ and values are the type of that column.
+
+ `FieldType` can be one of the following:
+ - [`Value`] feature specifies a single data type value, e.g. `int64` or `string`.
+ - [`ClassLabel`] feature specifies a predefined set of classes which can have labels associated to them and
+ will be stored as integers in the dataset.
+ - Python `dict` specifies a composite feature containing a mapping of sub-fields to sub-features.
+ It's possible to have nested fields of nested fields in an arbitrary manner.
+ - [`List`] or [`LargeList`] specifies a composite feature containing a sequence of
+ sub-features, all of the same feature type.
+ - [`Array2D`], [`Array3D`], [`Array4D`] or [`Array5D`] feature for multidimensional arrays.
+ - [`Audio`] feature to store the absolute path to an audio file or a dictionary with the relative path
+ to an audio file ("path" key) and its bytes content ("bytes" key).
+ This feature loads the audio lazily with a decoder.
+ - [`Image`] feature to store the absolute path to an image file, an `np.ndarray` object, a `PIL.Image.Image` object
+ or a dictionary with the relative path to an image file ("path" key) and its bytes content ("bytes" key).
+ This feature extracts the image data.
+ - [`Video`] feature to store the absolute path to a video file, a `torchcodec.decoders.VideoDecoder` object
+ or a dictionary with the relative path to a video file ("path" key) and its bytes content ("bytes" key).
+ This feature loads the video lazily with a decoder.
+ - [`Pdf`] feature to store the absolute path to a PDF file, a `pdfplumber.pdf.PDF` object
+ or a dictionary with the relative path to a PDF file ("path" key) and its bytes content ("bytes" key).
+ This feature loads the PDF lazily with a PDF reader.
+ - [`Nifti`] feature to store the absolute path to a NIfTI neuroimaging file, a `nibabel.Nifti1Image` object
+ or a dictionary with the relative path to a NIfTI file ("path" key) and its bytes content ("bytes" key).
+ This feature loads the NIfTI file lazily with nibabel.
+ - [`Translation`] or [`TranslationVariableLanguages`] feature specific to Machine Translation.
+ - [`Json`] feature to store unstructred data, e.g. containing mixed/abritrary types. Under the hood
+ """
+
+ def __init__(*args, **kwargs):
+ # self not in the signature to allow passing self as a kwarg
+ if not args:
+ raise TypeError("descriptor '__init__' of 'Features' object needs an argument")
+ self, *args = args
+ super(Features, self).__init__(*args, **kwargs)
+ # keep track of columns which require decoding
+ self._column_requires_decoding: dict[str, bool] = {
+ col: require_decoding(feature) for col, feature in self.items()
+ }
+
+ # backward compatibility with datasets<4 : [feature] -> List(feature)
+ def _check_old_list(feature):
+ if isinstance(feature, list):
+ return List(_visit(feature[0], _check_old_list))
+ return feature
+
+ for column_name, feature in self.items():
+ self[column_name] = _visit(feature, _check_old_list)
+
+ __setitem__ = keep_features_dicts_synced(dict.__setitem__)
+ __delitem__ = keep_features_dicts_synced(dict.__delitem__)
+ update = keep_features_dicts_synced(dict.update)
+ setdefault = keep_features_dicts_synced(dict.setdefault)
+ pop = keep_features_dicts_synced(dict.pop)
+ popitem = keep_features_dicts_synced(dict.popitem)
+ clear = keep_features_dicts_synced(dict.clear)
+
+ def __reduce__(self):
+ return Features, (dict(self),)
+
+ @property
+ def type(self):
+ """
+ Features field types.
+
+ Returns:
+ :obj:`pyarrow.DataType`
+ """
+ return get_nested_type(self)
+
+ @property
+ def arrow_schema(self):
+ """
+ Features schema.
+
+ Returns:
+ :obj:`pyarrow.Schema`
+ """
+ hf_metadata = {"info": {"features": self.to_dict()}}
+ return pa.schema(self.type).with_metadata({"huggingface": json.dumps(hf_metadata)})
+
+ @classmethod
+ def from_arrow_schema(cls, pa_schema: pa.Schema) -> "Features":
+ """
+ Construct [`Features`] from Arrow Schema.
+ It also checks the schema metadata for Hugging Face Datasets features.
+ Non-nullable fields are not supported and set to nullable.
+
+ Also, pa.dictionary is not supported and it uses its underlying type instead.
+ Therefore datasets convert DictionaryArray objects to their actual values.
+
+ Args:
+ pa_schema (`pyarrow.Schema`):
+ Arrow Schema.
+
+ Returns:
+ [`Features`]
+ """
+ # try to load features from the arrow schema metadata
+ metadata_features = Features()
+ if pa_schema.metadata is not None and b"huggingface" in pa_schema.metadata:
+ metadata = json.loads(pa_schema.metadata[b"huggingface"].decode())
+ if "info" in metadata and "features" in metadata["info"] and metadata["info"]["features"] is not None:
+ metadata_features = Features.from_dict(metadata["info"]["features"])
+ metadata_features_schema = metadata_features.arrow_schema
+ obj = {
+ field.name: (
+ metadata_features[field.name]
+ if field.name in metadata_features and metadata_features_schema.field(field.name) == field
+ else generate_from_arrow_type(field.type)
+ )
+ for field in pa_schema
+ }
+ return cls(**obj)
+
+ @classmethod
+ def from_dict(cls, dic) -> "Features":
+ """
+ Construct [`Features`] from dict.
+
+ Regenerate the nested feature object from a deserialized dict.
+ We use the `_type` key to infer the dataclass name of the feature `FieldType`.
+
+ It allows for a convenient constructor syntax
+ to define features from deserialized JSON dictionaries. This function is used in particular when deserializing
+ a [`DatasetInfo`] that was dumped to a JSON object. This acts as an analogue to
+ [`Features.from_arrow_schema`] and handles the recursive field-by-field instantiation, but doesn't require
+ any mapping to/from pyarrow, except for the fact that it takes advantage of the mapping of pyarrow primitive
+ dtypes that [`Value`] automatically performs.
+
+ Args:
+ dic (`dict[str, Any]`):
+ Python dictionary.
+
+ Returns:
+ `Features`
+
+ Example::
+ >>> Features.from_dict({'_type': {'dtype': 'string', 'id': None, '_type': 'Value'}})
+ {'_type': Value('string')}
+ """
+ obj = generate_from_dict(dic)
+ return cls(**obj)
+
+ def to_dict(self):
+ return asdict(self)
+
+ def _to_yaml_list(self) -> list:
+ # we compute the YAML list from the dict representation that is used for JSON dump
+ yaml_data = self.to_dict()
+
+ def simplify(feature: dict) -> dict:
+ if not isinstance(feature, dict):
+ raise TypeError(f"Expected a dict but got a {type(feature)}: {feature}")
+
+ for list_type in ["large_list", "list", "sequence"]:
+ #
+ # list_type: -> list_type: int32
+ # dtype: int32 ->
+ #
+ if isinstance(feature.get(list_type), dict) and list(feature[list_type]) == ["dtype"]:
+ feature[list_type] = feature[list_type]["dtype"]
+
+ #
+ # list_type: -> list_type:
+ # struct: -> - name: foo
+ # - name: foo -> dtype: int32
+ # dtype: int32 ->
+ #
+ if isinstance(feature.get(list_type), dict) and list(feature[list_type]) == ["struct"]:
+ feature[list_type] = feature[list_type]["struct"]
+
+ #
+ # class_label: -> class_label:
+ # names: -> names:
+ # - negative -> '0': negative
+ # - positive -> '1': positive
+ #
+ if isinstance(feature.get("class_label"), dict) and isinstance(feature["class_label"].get("names"), list):
+ # server-side requirement: keys must be strings
+ feature["class_label"]["names"] = {
+ str(label_id): label_name for label_id, label_name in enumerate(feature["class_label"]["names"])
+ }
+ return feature
+
+ def to_yaml_inner(obj: Union[dict, list]) -> dict:
+ if isinstance(obj, dict):
+ _type = obj.pop("_type", None)
+ if _type == "LargeList":
+ _feature = obj.pop("feature")
+ return simplify({"large_list": to_yaml_inner(_feature), **obj})
+ elif _type == "List":
+ _feature = obj.pop("feature")
+ return simplify({"list": to_yaml_inner(_feature), **obj})
+ elif _type == "Value":
+ return obj
+ elif _type and not obj:
+ return {"dtype": camelcase_to_snakecase(_type)}
+ elif _type:
+ return {"dtype": simplify({camelcase_to_snakecase(_type): obj})}
+ else:
+ return {"struct": [{"name": name, **to_yaml_inner(_feature)} for name, _feature in obj.items()]}
+ elif isinstance(obj, list):
+ return simplify({"list": simplify(to_yaml_inner(obj[0]))})
+ elif isinstance(obj, tuple):
+ return to_yaml_inner(list(obj))
+ else:
+ raise TypeError(f"Expected a dict or a list but got {type(obj)}: {obj}")
+
+ def to_yaml_types(obj: dict) -> dict:
+ if isinstance(obj, dict):
+ return {k: to_yaml_types(v) for k, v in obj.items()}
+ elif isinstance(obj, list):
+ return [to_yaml_types(v) for v in obj]
+ elif isinstance(obj, tuple):
+ return to_yaml_types(list(obj))
+ else:
+ return obj
+
+ return to_yaml_types(to_yaml_inner(yaml_data)["struct"])
+
+ @classmethod
+ def _from_yaml_list(cls, yaml_data: list) -> "Features":
+ yaml_data = copy.deepcopy(yaml_data)
+
+ # we convert the list obtained from YAML data into the dict representation that is used for JSON dump
+
+ def unsimplify(feature: dict) -> dict:
+ if not isinstance(feature, dict):
+ raise TypeError(f"Expected a dict but got a {type(feature)}: {feature}")
+
+ for list_type in ["large_list", "list", "sequence"]:
+ #
+ # list_type: int32 -> list_type:
+ # -> dtype: int32
+ #
+ if isinstance(feature.get(list_type), str):
+ feature[list_type] = {"dtype": feature[list_type]}
+
+ #
+ # class_label: -> class_label:
+ # names: -> names:
+ # '0': negative -> - negative
+ # '1': positive -> - positive
+ #
+ if isinstance(feature.get("class_label"), dict) and isinstance(feature["class_label"].get("names"), dict):
+ label_ids = sorted(feature["class_label"]["names"], key=int)
+ if label_ids and [int(label_id) for label_id in label_ids] != list(range(int(label_ids[-1]) + 1)):
+ raise ValueError(
+ f"ClassLabel expected a value for all label ids [0:{int(label_ids[-1]) + 1}] but some ids are missing."
+ )
+ feature["class_label"]["names"] = [feature["class_label"]["names"][label_id] for label_id in label_ids]
+ return feature
+
+ def from_yaml_inner(obj: Union[dict, list]) -> Union[dict, list]:
+ if isinstance(obj, dict):
+ if not obj:
+ return {}
+ _type = next(iter(obj))
+ if _type == "large_list":
+ _feature = from_yaml_inner(unsimplify(obj).pop(_type))
+ return {"feature": _feature, **obj, "_type": "LargeList"}
+ if _type == "sequence": # backward compatibility
+ if isinstance(obj[_type], list):
+ _feature = from_yaml_inner(unsimplify(obj).pop(_type))
+ return {
+ name: {"feature": _subfeature, **obj, "_type": "List"}
+ for name, _subfeature in _feature.items()
+ }
+ else:
+ _feature = from_yaml_inner(unsimplify(obj).pop(_type))
+ return {"feature": _feature, **obj, "_type": "List"}
+ if _type == "list":
+ _feature = from_yaml_inner(unsimplify(obj).pop(_type))
+ return {"feature": _feature, **obj, "_type": "List"}
+ if _type == "struct":
+ return from_yaml_inner(obj["struct"])
+ elif _type == "dtype":
+ if isinstance(obj["dtype"], str):
+ # e.g. int32, float64, string, audio, image
+ try:
+ Value(obj["dtype"])
+ return {**obj, "_type": "Value"}
+ except ValueError:
+ # e.g. Audio, Image, ArrayXD
+ return {"_type": snakecase_to_camelcase(obj["dtype"])}
+ else:
+ return from_yaml_inner(obj["dtype"])
+ else:
+ return {"_type": snakecase_to_camelcase(_type), **unsimplify(obj)[_type]}
+ elif isinstance(obj, list):
+ names = [_feature.pop("name") for _feature in obj]
+ return {name: from_yaml_inner(_feature) for name, _feature in zip(names, obj)}
+ else:
+ raise TypeError(f"Expected a dict or a list but got {type(obj)}: {obj}")
+
+ return cls.from_dict(from_yaml_inner(yaml_data))
+
+ def encode_example(self, example):
+ """
+ Encode example into a format for Arrow.
+
+ Args:
+ example (`dict[str, Any]`):
+ Data in a Dataset row.
+
+ Returns:
+ `dict[str, Any]`
+ """
+ example = cast_to_python_objects(example)
+ return encode_nested_example(self, example)
+
+ def encode_column(self, column, column_name: str):
+ """
+ Encode column into a format for Arrow.
+
+ Args:
+ column (`list[Any]`):
+ Data in a Dataset column.
+ column_name (`str`):
+ Dataset column name.
+
+ Returns:
+ `list[Any]`
+ """
+ column = cast_to_python_objects(column)
+ return [encode_nested_example(self[column_name], obj, level=1) for obj in column]
+
+ def encode_batch(self, batch):
+ """
+ Encode batch into a format for Arrow.
+
+ Args:
+ batch (`dict[str, list[Any]]`):
+ Data in a Dataset batch.
+
+ Returns:
+ `dict[str, list[Any]]`
+ """
+ encoded_batch = {}
+ if set(batch) != set(self):
+ raise ValueError(f"Column mismatch between batch {set(batch)} and features {set(self)}")
+ for key, column in batch.items():
+ column = cast_to_python_objects(column)
+ encoded_batch[key] = [encode_nested_example(self[key], obj, level=1) for obj in column]
+ return encoded_batch
+
+ def decode_example(self, example: dict, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None):
+ """Decode example with custom feature decoding.
+
+ Args:
+ example (`dict[str, Any]`):
+ Dataset row data.
+ token_per_repo_id (`dict`, *optional*):
+ To access and decode audio or image files from private repositories on the Hub, you can pass
+ a dictionary `repo_id (str) -> token (bool or str)`.
+
+ Returns:
+ `dict[str, Any]`
+ """
+
+ return {
+ column_name: decode_nested_example(feature, value, token_per_repo_id=token_per_repo_id)
+ if self._column_requires_decoding[column_name]
+ else value
+ for column_name, (feature, value) in zip_dict(
+ {key: value for key, value in self.items() if key in example}, example
+ )
+ }
+
+ def decode_column(
+ self, column: list, column_name: str, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None
+ ):
+ """Decode column with custom feature decoding.
+
+ Args:
+ column (`list[Any]`):
+ Dataset column data.
+ column_name (`str`):
+ Dataset column name.
+
+ Returns:
+ `list[Any]`
+ """
+ return (
+ [
+ decode_nested_example(self[column_name], value, token_per_repo_id=token_per_repo_id)
+ if value is not None
+ else None
+ for value in column
+ ]
+ if self._column_requires_decoding[column_name]
+ else column
+ )
+
+ def decode_batch(self, batch: dict, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None):
+ """Decode batch with custom feature decoding.
+
+ Args:
+ batch (`dict[str, list[Any]]`):
+ Dataset batch data.
+ token_per_repo_id (`dict`, *optional*):
+ To access and decode audio or image files from private repositories on the Hub, you can pass
+ a dictionary repo_id (str) -> token (bool or str)
+
+ Returns:
+ `dict[str, list[Any]]`
+ """
+ decoded_batch = {}
+ for column_name, column in batch.items():
+ decoded_batch[column_name] = (
+ [
+ decode_nested_example(self[column_name], value, token_per_repo_id=token_per_repo_id)
+ if value is not None
+ else None
+ for value in column
+ ]
+ if self._column_requires_decoding[column_name]
+ else column
+ )
+ return decoded_batch
+
+ def copy(self) -> "Features":
+ """
+ Make a deep copy of [`Features`].
+
+ Returns:
+ [`Features`]
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
+ >>> copy_of_features = ds.features.copy()
+ >>> copy_of_features
+ {'label': ClassLabel(names=['neg', 'pos']),
+ 'text': Value('string')}
+ ```
+ """
+ return copy.deepcopy(self)
+
+ def reorder_fields_as(self, other: "Features") -> "Features":
+ """
+ Reorder Features fields to match the field order of other [`Features`].
+
+ The order of the fields is important since it matters for the underlying arrow data.
+ Re-ordering the fields allows to make the underlying arrow data type match.
+
+ Args:
+ other ([`Features`]):
+ The other [`Features`] to align with.
+
+ Returns:
+ [`Features`]
+
+ Example::
+
+ >>> from datasets import Features, List, Value
+ >>> # let's say we have two features with a different order of nested fields (for a and b for example)
+ >>> f1 = Features({"root": {"a": Value("string"), "b": Value("string")}})
+ >>> f2 = Features({"root": {"b": Value("string"), "a": Value("string")}})
+ >>> assert f1.type != f2.type
+ >>> # re-ordering keeps the base structure (here List is defined at the root level), but makes the fields order match
+ >>> f1.reorder_fields_as(f2)
+ {'root': List({'b': Value('string'), 'a': Value('string')})}
+ >>> assert f1.reorder_fields_as(f2).type == f2.type
+ """
+
+ def recursive_reorder(source, target, stack=""):
+ stack_position = " at " + stack[1:] if stack else ""
+ if isinstance(source, dict):
+ if not isinstance(target, dict):
+ raise ValueError(f"Type mismatch: between {source} and {target}" + stack_position)
+ if sorted(source) != sorted(target):
+ message = (
+ f"Keys mismatch: between {source} (source) and {target} (target).\n"
+ f"{source.keys() - target.keys()} are missing from target "
+ f"and {target.keys() - source.keys()} are missing from source" + stack_position
+ )
+ raise ValueError(message)
+ return {key: recursive_reorder(source[key], target[key], stack + f".{key}") for key in target}
+ elif isinstance(source, List):
+ if not isinstance(target, List):
+ raise ValueError(f"Type mismatch: between {source} and {target}" + stack_position)
+ return List(recursive_reorder(source.feature, target.feature, stack + "."), length=source.length)
+ elif isinstance(source, LargeList):
+ if not isinstance(target, LargeList):
+ raise ValueError(f"Type mismatch: between {source} and {target}" + stack_position)
+ return LargeList(recursive_reorder(source.feature, target.feature, stack + "."))
+ else:
+ return source
+
+ return Features(recursive_reorder(self, other))
+
+ def flatten(self, max_depth=16) -> "Features":
+ """Flatten the features. Every dictionary column is removed and is replaced by
+ all the subfields it contains. The new fields are named by concatenating the
+ name of the original column and the subfield name like this: `.`.
+
+ If a column contains nested dictionaries, then all the lower-level subfields names are
+ also concatenated to form new columns: `..`, etc.
+
+ Returns:
+ [`Features`]:
+ The flattened features.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("rajpurkar/squad", split="train")
+ >>> ds.features.flatten()
+ {'answers.answer_start': List(Value('int32'), id=None),
+ 'answers.text': List(Value('string'), id=None),
+ 'context': Value('string'),
+ 'id': Value('string'),
+ 'question': Value('string'),
+ 'title': Value('string')}
+ ```
+ """
+ for depth in range(1, max_depth):
+ no_change = True
+ flattened = self.copy()
+ for column_name, subfeature in self.items():
+ if isinstance(subfeature, dict):
+ no_change = False
+ flattened.update({f"{column_name}.{k}": v for k, v in subfeature.items()})
+ del flattened[column_name]
+ elif hasattr(subfeature, "flatten") and subfeature.flatten() != subfeature:
+ no_change = False
+ flattened.update({f"{column_name}.{k}": v for k, v in subfeature.flatten().items()})
+ del flattened[column_name]
+ self = flattened
+ if no_change:
+ break
+ return self
+
+
+def _is_null_feature(feature) -> bool:
+ """Recursively check if a feature represents a null type.
+
+ This handles not only top-level ``Value("null")`` but also nested null types
+ such as ``List(Value("null"))``, ``LargeList(Value("null"))``, and
+ ``Sequence(Value("null"))``, which can arise when a shard contains only
+ empty lists during multi-process ``Dataset.map()``.
+ """
+ if isinstance(feature, Value) and feature.dtype == "null":
+ return True
+ if isinstance(feature, (Sequence, LargeList)) and hasattr(feature, "feature"):
+ return _is_null_feature(feature.feature)
+ return False
+
+
+def _align_features(features_list: list[Features]) -> list[Features]:
+ """Align dictionaries of features so that the keys that are found in multiple dictionaries share the same feature."""
+ name2feature = {}
+ for features in features_list:
+ for k, v in features.items():
+ if k in name2feature and isinstance(v, dict):
+ # Recursively align features.
+ name2feature[k] = _align_features([name2feature[k], v])[0]
+ elif k not in name2feature or _is_null_feature(name2feature[k]):
+ name2feature[k] = v
+
+ return [Features({k: name2feature[k] for k in features.keys()}) for features in features_list]
+
+
+def _check_if_features_can_be_aligned(features_list: list[Features]):
+ """Check if the dictionaries of features can be aligned.
+
+ Two dictionaries of features can be aligned if the keys they share have the same type or some of them is of
+ type ``Value("null")`` (or a container wrapping ``Value("null")``, such as ``List(Value("null"))``).
+ """
+ name2feature = {}
+ for features in features_list:
+ for k, v in features.items():
+ if k not in name2feature or _is_null_feature(name2feature[k]):
+ name2feature[k] = v
+
+ for features in features_list:
+ for k, v in features.items():
+ if isinstance(v, dict) and isinstance(name2feature[k], dict):
+ # Deep checks for structure.
+ _check_if_features_can_be_aligned([name2feature[k], v])
+ elif not _is_null_feature(v) and name2feature[k] != v:
+ raise ValueError(
+ f'The features can\'t be aligned because the key {k} of features {features} has unexpected type - {v} (expected either {name2feature[k]} or Value("null").'
+ )
+
+
+def _fix_for_backward_compatible_features(feature: Any) -> FeatureType:
+ def _fix_old_list(feature):
+ if isinstance(feature, list):
+ return List(_fix_for_backward_compatible_features(feature[0]))
+ return feature
+
+ return _visit(feature, _fix_old_list)
diff --git a/venv/Lib/site-packages/datasets/features/image.py b/venv/Lib/site-packages/datasets/features/image.py
new file mode 100644
index 0000000000000000000000000000000000000000..9a7953dd6bc4b6638de7d53d23764a996964e75e
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/features/image.py
@@ -0,0 +1,441 @@
+import os
+import sys
+import warnings
+from dataclasses import dataclass, field
+from io import BytesIO
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union
+
+import numpy as np
+import pyarrow as pa
+
+from .. import config
+from ..download.download_config import DownloadConfig
+from ..table import array_cast
+from ..utils.file_utils import is_local_path, is_remote_url, xopen
+from ..utils.py_utils import first_non_null_value, no_op_if_value_is_null, string_to_dict
+
+
+if TYPE_CHECKING:
+ import PIL.Image
+
+ from .features import FeatureType
+
+
+_IMAGE_COMPRESSION_FORMATS: Optional[list[str]] = None
+_NATIVE_BYTEORDER = "<" if sys.byteorder == "little" else ">"
+# Origin: https://github.com/python-pillow/Pillow/blob/698951e19e19972aeed56df686868f1329981c12/src/PIL/Image.py#L3126 minus "|i1" which values are not preserved correctly when saving and loading an image
+_VALID_IMAGE_ARRAY_DTPYES = [
+ np.dtype("|b1"),
+ np.dtype("|u1"),
+ np.dtype("u2"),
+ np.dtype("i2"),
+ np.dtype("u4"),
+ np.dtype("i4"),
+ np.dtype("f4"),
+ np.dtype("f8"),
+]
+
+
+@dataclass
+class Image:
+ """Image [`Feature`] to read image data from an image file.
+
+ Input: The Image feature accepts as input:
+ - A `str`: Absolute path to the image file (i.e. random access is allowed).
+ - A `pathlib.Path`: path to the image file (i.e. random access is allowed).
+ - A `dict` with the keys:
+
+ - `path`: String with relative path of the image file to the archive file.
+ - `bytes`: Bytes of the image file.
+
+ This is useful for parquet or webdataset files which embed image files.
+
+ - An `np.ndarray`: NumPy array representing an image.
+ - A `PIL.Image.Image`: PIL image object.
+
+ Output: The Image features output data as `PIL.Image.Image` objects.
+
+ Args:
+ mode (`str`, *optional*):
+ The mode to convert the image to. If `None`, the native mode of the image is used.
+ decode (`bool`, defaults to `True`):
+ Whether to decode the image data. If `False`,
+ returns the underlying dictionary in the format `{"path": image_path, "bytes": image_bytes}`.
+
+ Examples:
+
+ ```py
+ >>> from datasets import load_dataset, Image
+ >>> ds = load_dataset("AI-Lab-Makerere/beans", split="train")
+ >>> ds.features["image"]
+ Image(decode=True, id=None)
+ >>> ds[0]["image"]
+
+ >>> ds = ds.cast_column('image', Image(decode=False))
+ {'bytes': None,
+ 'path': '/root/.cache/huggingface/datasets/downloads/extracted/b0a21163f78769a2cf11f58dfc767fb458fc7cea5c05dccc0144a2c0f0bc1292/train/healthy/healthy_train.85.jpg'}
+ ```
+ """
+
+ mode: Optional[str] = None
+ decode: bool = True
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ dtype: ClassVar[str] = "PIL.Image.Image"
+ pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()})
+ _type: str = field(default="Image", init=False, repr=False)
+
+ def __call__(self):
+ return self.pa_type
+
+ def encode_example(self, value: Union[str, bytes, bytearray, dict, np.ndarray, "PIL.Image.Image"]) -> dict:
+ """Encode example into a format for Arrow.
+
+ Args:
+ value (`str`, `np.ndarray`, `PIL.Image.Image` or `dict`):
+ Data passed as input to Image feature.
+
+ Returns:
+ `dict` with "path" and "bytes" fields
+ """
+ if config.PIL_AVAILABLE:
+ import PIL.Image
+ else:
+ raise ImportError("To support encoding images, please install 'Pillow'.")
+
+ if isinstance(value, list):
+ value = np.array(value)
+
+ if isinstance(value, str):
+ return {"path": value, "bytes": None}
+ elif isinstance(value, Path):
+ return {"path": str(value.absolute()), "bytes": None}
+ elif isinstance(value, (bytes, bytearray)):
+ return {"path": None, "bytes": value}
+ elif isinstance(value, np.ndarray):
+ # convert the image array to PNG/TIFF bytes
+ return encode_np_array(value)
+ elif isinstance(value, PIL.Image.Image):
+ # convert the PIL image to bytes (default format is PNG/TIFF)
+ return encode_pil_image(value)
+ elif value.get("path") is not None and os.path.isfile(value["path"]):
+ # we set "bytes": None to not duplicate the data if they're already available locally
+ return {"bytes": None, "path": value.get("path")}
+ elif value.get("bytes") is not None or value.get("path") is not None:
+ # store the image bytes, and path is used to infer the image format using the file extension
+ return {"bytes": value.get("bytes"), "path": value.get("path")}
+ else:
+ raise ValueError(
+ f"An image sample should have one of 'path' or 'bytes' but they are missing or None in {value}."
+ )
+
+ def decode_example(self, value: dict, token_per_repo_id=None) -> "PIL.Image.Image":
+ """Decode example image file into image data.
+
+ Args:
+ value (`str` or `dict`):
+ A string with the absolute image file path, a dictionary with
+ keys:
+
+ - `path`: String with absolute or relative image file path.
+ - `bytes`: The bytes of the image file.
+ token_per_repo_id (`dict`, *optional*):
+ To access and decode
+ image files from private repositories on the Hub, you can pass
+ a dictionary repo_id (`str`) -> token (`bool` or `str`).
+
+ Returns:
+ `PIL.Image.Image`
+ """
+ if not self.decode:
+ raise RuntimeError("Decoding is disabled for this feature. Please use Image(decode=True) instead.")
+
+ if config.PIL_AVAILABLE:
+ import PIL.Image
+ import PIL.ImageOps
+ else:
+ raise ImportError("To support decoding images, please install 'Pillow'.")
+
+ if token_per_repo_id is None:
+ token_per_repo_id = {}
+
+ path, bytes_ = value["path"], value["bytes"]
+ if bytes_ is None:
+ if path is None:
+ raise ValueError(f"An image should have one of 'path' or 'bytes' but both are None in {value}.")
+ else:
+ if is_local_path(path):
+ image = PIL.Image.open(path)
+ else:
+ source_url = path.split("::")[-1]
+ pattern = (
+ config.HUB_DATASETS_URL
+ if source_url.startswith(config.HF_ENDPOINT)
+ else config.HUB_DATASETS_HFFS_URL
+ )
+ source_url_fields = string_to_dict(source_url, pattern)
+ token = (
+ token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None
+ )
+ download_config = DownloadConfig(token=token)
+ with xopen(path, "rb", download_config=download_config) as f:
+ bytes_ = BytesIO(f.read())
+ image = PIL.Image.open(bytes_)
+ else:
+ image = PIL.Image.open(BytesIO(bytes_))
+ image.load() # to avoid "Too many open files" errors
+ if image.getexif().get(PIL.Image.ExifTags.Base.Orientation) is not None:
+ image = PIL.ImageOps.exif_transpose(image)
+ if self.mode and self.mode != image.mode:
+ image = image.convert(self.mode)
+ return image
+
+ def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]:
+ """If in the decodable state, return the feature itself, otherwise flatten the feature into a dictionary."""
+ from .features import Value
+
+ return (
+ self
+ if self.decode
+ else {
+ "bytes": Value("binary"),
+ "path": Value("string"),
+ }
+ )
+
+ def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray, pa.ListArray]) -> pa.StructArray:
+ """Cast an Arrow array to the Image arrow storage type.
+ The Arrow types that can be converted to the Image pyarrow storage type are:
+
+ - `pa.string()` - it must contain the "path" data
+ - `pa.large_string()` - it must contain the "path" data (will be cast to string if possible)
+ - `pa.binary()` - it must contain the image bytes
+ - `pa.struct({"bytes": pa.binary()})`
+ - `pa.struct({"path": pa.string()})`
+ - `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter
+ - `pa.list(*)` - it must contain the image array data
+
+ Args:
+ storage (`Union[pa.StringArray, pa.StructArray, pa.ListArray]`):
+ PyArrow array to cast.
+
+ Returns:
+ `pa.StructArray`: Array in the Image arrow storage type, that is
+ `pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
+ """
+ if pa.types.is_large_string(storage.type):
+ try:
+ storage = storage.cast(pa.string())
+ except pa.ArrowInvalid as e:
+ raise ValueError(
+ f"Failed to cast large_string to string for Image feature. "
+ f"This can happen if string values exceed 2GB. "
+ f"Original error: {e}"
+ ) from e
+ if pa.types.is_string(storage.type):
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
+ storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_large_binary(storage.type):
+ storage = array_cast(
+ storage, pa.binary()
+ ) # this can fail in case of big images, paths should be used instead
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_binary(storage.type):
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_struct(storage.type):
+ if storage.type.get_field_index("bytes") >= 0:
+ bytes_array = storage.field("bytes")
+ else:
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
+ if storage.type.get_field_index("path") >= 0:
+ path_array = storage.field("path")
+ else:
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_list(storage.type):
+ bytes_array = pa.array(
+ [encode_np_array(np.array(arr))["bytes"] if arr is not None else None for arr in storage.to_pylist()],
+ type=pa.binary(),
+ )
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays(
+ [bytes_array, path_array], ["bytes", "path"], mask=bytes_array.is_null()
+ )
+ return array_cast(storage, self.pa_type)
+
+ def embed_storage(
+ self, storage: pa.StructArray, token_per_repo_id=None, local_files: bool = True, remote_files: bool = True
+ ) -> pa.StructArray:
+ """Embed image files into the Arrow array.
+
+ Args:
+ storage (`pa.StructArray`):
+ PyArrow array to embed.
+ token_per_repo_id (`dict`, optional):
+ Dictionary repo_id -> token to fetch the files bytes.
+ local_files (`bool`, defaults to `True`)
+ Whether to embed local files data in the array
+
+
+ remote_files (`bool`, defaults to `True`)
+ Whether to embed remote files data in the array.
+ E.g. files with paths that start with hf:// or https://
+
+
+
+ Returns:
+ `pa.StructArray`: Array in the Image arrow storage type, that is
+ `pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
+ """
+ if token_per_repo_id is None:
+ token_per_repo_id = {}
+
+ @no_op_if_value_is_null
+ def path_to_bytes(path):
+ source_url = path.split("::")[-1]
+ pattern = (
+ config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL
+ )
+ source_url_fields = string_to_dict(source_url, pattern)
+ token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None
+ download_config = DownloadConfig(token=token)
+ with xopen(path, "rb", download_config=download_config) as f:
+ return f.read()
+
+ bytes_array = pa.array(
+ [
+ (
+ path_to_bytes(x["path"])
+ if x["bytes"] is None
+ and ((local_files and is_local_path(x["path"])) or (remote_files and is_remote_url(x["path"])))
+ else x["bytes"]
+ )
+ if x is not None
+ else None
+ for x in storage.to_pylist()
+ ],
+ type=pa.binary(),
+ )
+ path_array = pa.array(
+ [
+ (
+ os.path.basename(path)
+ if (local_files and is_local_path(path)) or (remote_files and is_remote_url(path))
+ else path
+ )
+ if path is not None
+ else None
+ for path in storage.field("path").to_pylist()
+ ],
+ type=pa.string(),
+ )
+ storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null())
+ return array_cast(storage, self.pa_type)
+
+
+def list_image_compression_formats() -> list[str]:
+ if config.PIL_AVAILABLE:
+ import PIL.Image
+ else:
+ raise ImportError("To support encoding images, please install 'Pillow'.")
+
+ global _IMAGE_COMPRESSION_FORMATS
+ if _IMAGE_COMPRESSION_FORMATS is None:
+ PIL.Image.init()
+ _IMAGE_COMPRESSION_FORMATS = list(set(PIL.Image.OPEN.keys()) & set(PIL.Image.SAVE.keys()))
+ return _IMAGE_COMPRESSION_FORMATS
+
+
+def image_to_bytes(image: "PIL.Image.Image") -> bytes:
+ """Convert a PIL Image object to bytes using native compression if possible, otherwise use PNG/TIFF compression."""
+ buffer = BytesIO()
+ if image.format in list_image_compression_formats():
+ format = image.format
+ else:
+ format = "PNG" if image.mode in ["1", "L", "LA", "RGB", "RGBA"] else "TIFF"
+ image.save(buffer, format=format)
+ return buffer.getvalue()
+
+
+def encode_pil_image(image: "PIL.Image.Image") -> dict:
+ if hasattr(image, "filename") and image.filename != "":
+ return {"path": image.filename, "bytes": None}
+ else:
+ return {"path": None, "bytes": image_to_bytes(image)}
+
+
+def encode_np_array(array: np.ndarray) -> dict:
+ if config.PIL_AVAILABLE:
+ import PIL.Image
+ else:
+ raise ImportError("To support encoding images, please install 'Pillow'.")
+
+ dtype = array.dtype
+ dtype_byteorder = dtype.byteorder if dtype.byteorder != "=" else _NATIVE_BYTEORDER
+ dtype_kind = dtype.kind
+ dtype_itemsize = dtype.itemsize
+
+ dest_dtype = None
+
+ # Multi-channel array case (only np.dtype("|u1") is allowed)
+ if array.shape[2:]:
+ if dtype_kind not in ["u", "i"]:
+ raise TypeError(
+ f"Unsupported array dtype {dtype} for image encoding. Only {dest_dtype} is supported for multi-channel arrays."
+ )
+ dest_dtype = np.dtype("|u1")
+ if dtype != dest_dtype:
+ warnings.warn(f"Downcasting array dtype {dtype} to {dest_dtype} to be compatible with 'Pillow'")
+ # Exact match
+ elif dtype in _VALID_IMAGE_ARRAY_DTPYES:
+ dest_dtype = dtype
+ else: # Downcast the type within the kind (np.can_cast(from_type, to_type, casting="same_kind") doesn't behave as expected, so do it manually)
+ while dtype_itemsize >= 1:
+ dtype_str = dtype_byteorder + dtype_kind + str(dtype_itemsize)
+ if np.dtype(dtype_str) in _VALID_IMAGE_ARRAY_DTPYES:
+ dest_dtype = np.dtype(dtype_str)
+ warnings.warn(f"Downcasting array dtype {dtype} to {dest_dtype} to be compatible with 'Pillow'")
+ break
+ else:
+ dtype_itemsize //= 2
+ if dest_dtype is None:
+ raise TypeError(
+ f"Cannot downcast dtype {dtype} to a valid image dtype. Valid image dtypes: {_VALID_IMAGE_ARRAY_DTPYES}"
+ )
+
+ image = PIL.Image.fromarray(array.astype(dest_dtype))
+ return {"path": None, "bytes": image_to_bytes(image)}
+
+
+def objects_to_list_of_image_dicts(
+ objs: Union[list[str], list[dict], list[np.ndarray], list["PIL.Image.Image"]],
+) -> list[dict]:
+ """Encode a list of objects into a format suitable for creating an extension array of type `ImageExtensionType`."""
+ if config.PIL_AVAILABLE:
+ import PIL.Image
+ else:
+ raise ImportError("To support encoding images, please install 'Pillow'.")
+
+ if objs:
+ _, obj = first_non_null_value(objs)
+ if isinstance(obj, str):
+ return [{"path": obj, "bytes": None} if obj is not None else None for obj in objs]
+ if isinstance(obj, np.ndarray):
+ obj_to_image_dict_func = no_op_if_value_is_null(encode_np_array)
+ return [obj_to_image_dict_func(obj) for obj in objs]
+ elif isinstance(obj, PIL.Image.Image):
+ obj_to_image_dict_func = no_op_if_value_is_null(encode_pil_image)
+ return [obj_to_image_dict_func(obj) for obj in objs]
+ else:
+ return objs
+ else:
+ return objs
diff --git a/venv/Lib/site-packages/datasets/features/mesh.py b/venv/Lib/site-packages/datasets/features/mesh.py
new file mode 100644
index 0000000000000000000000000000000000000000..58da5d21d16942d68063afda2a037f576bac09c1
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/features/mesh.py
@@ -0,0 +1,295 @@
+import os
+from dataclasses import dataclass, field
+from io import BytesIO
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union
+
+import pyarrow as pa
+
+from .. import config
+from ..download.download_config import DownloadConfig
+from ..table import array_cast
+from ..utils.file_utils import is_local_path, is_remote_url, xopen
+from ..utils.py_utils import string_to_dict
+
+
+if TYPE_CHECKING:
+ import trimesh
+
+ from .features import FeatureType
+
+
+@dataclass
+class Mesh:
+ """Mesh [`Feature`] to read 3D mesh data from a file.
+
+ Input: The Mesh feature accepts as input:
+ - A `str`: Absolute path to the mesh file (i.e. random access is allowed).
+ - A `pathlib.Path`: path to the mesh file (i.e. random access is allowed).
+ - A `dict` with the keys:
+
+ - `path`: String with relative path of the mesh file to the archive file.
+ - `bytes`: Bytes of the mesh file.
+
+ This is useful for parquet or webdataset files which embed mesh files.
+
+ - A `trimesh.Trimesh` or `trimesh.Scene`: 3D mesh or scene object.
+
+ Output: The Mesh feature outputs data as `trimesh.Trimesh` or `trimesh.Scene` objects.
+
+ Args:
+ decode (`bool`, defaults to `True`):
+ Whether to decode the mesh data. If `False`,
+ returns the underlying dictionary in the format `{"path": mesh_path, "bytes": mesh_bytes}`.
+ Mesh decoding uses `trimesh` and supports `.glb`, `.ply`, and `.stl` files.
+ """
+
+ decode: bool = True
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ dtype: ClassVar[str] = "trimesh.Trimesh"
+ pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()})
+ _type: str = field(default="Mesh", init=False, repr=False)
+
+ def __call__(self):
+ return self.pa_type
+
+ def encode_example(self, value: Union[str, bytes, bytearray, dict, "trimesh.Trimesh", "trimesh.Scene"]) -> dict:
+ """Encode example into a format for Arrow.
+
+ Args:
+ value (`str`, `bytes`, `dict`, `trimesh.Trimesh`, or `trimesh.Scene`):
+ Data passed as input to Mesh feature.
+
+ Returns:
+ `dict` with "path" and "bytes" fields
+ """
+ if config.TRIMESH_AVAILABLE:
+ import trimesh
+ else:
+ trimesh = None
+
+ if isinstance(value, str):
+ return {"path": value, "bytes": None}
+ elif isinstance(value, Path):
+ return {"path": str(value.absolute()), "bytes": None}
+ elif isinstance(value, (bytes, bytearray)):
+ return {"path": None, "bytes": value}
+ elif trimesh is not None and isinstance(value, (trimesh.Trimesh, trimesh.Scene)):
+ return encode_trimesh_mesh(value)
+ elif isinstance(value, dict) and value.get("path") is not None and os.path.isfile(value["path"]):
+ # we set "bytes": None to not duplicate the data if they're already available locally
+ return {"bytes": None, "path": value.get("path")}
+ elif isinstance(value, dict) and (value.get("bytes") is not None or value.get("path") is not None):
+ # store the mesh bytes, and path is used to infer the mesh format using the file extension
+ return {"bytes": value.get("bytes"), "path": value.get("path")}
+ else:
+ raise ValueError(
+ f"A mesh sample should have one of 'path' or 'bytes' but they are missing or None in {value}."
+ )
+
+ def decode_example(self, value: dict, token_per_repo_id=None) -> Union["trimesh.Trimesh", "trimesh.Scene"]:
+ """Decode example mesh file.
+
+ Args:
+ value (`dict`):
+ A dictionary with keys:
+
+ - `path`: String with absolute or relative mesh file path.
+ - `bytes`: The bytes of the mesh file.
+ token_per_repo_id (`dict`, *optional*):
+ To access and decode
+ mesh files from private repositories on the Hub, you can pass
+ a dictionary repo_id (`str`) -> token (`bool` or `str`).
+
+ Returns:
+ `trimesh.Trimesh` or `trimesh.Scene`
+ """
+ if not self.decode:
+ raise RuntimeError("Decoding is disabled for this feature. Please use Mesh(decode=True) instead.")
+
+ if config.TRIMESH_AVAILABLE:
+ import trimesh
+ else:
+ raise ImportError("To support decoding meshes, please install 'trimesh'.")
+
+ if token_per_repo_id is None:
+ token_per_repo_id = {}
+
+ path, bytes_ = value["path"], value["bytes"]
+ if bytes_ is None:
+ if path is None:
+ raise ValueError(f"A mesh should have one of 'path' or 'bytes' but both are None in {value}.")
+ if is_local_path(path):
+ file_type = _infer_mesh_file_type(path)
+ if file_type is None:
+ raise ValueError("A mesh path should have a .glb, .ply, or .stl extension.")
+ return trimesh.load(path, file_type=file_type)
+ source_url = path.split("::")[-1]
+ pattern = (
+ config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL
+ )
+ source_url_fields = string_to_dict(source_url, pattern)
+ token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None
+ download_config = DownloadConfig(token=token)
+ with xopen(path, "rb", download_config=download_config) as f:
+ bytes_ = f.read()
+
+ file_type = _infer_mesh_file_type(path)
+ if file_type is None:
+ raise ValueError(
+ "Decoding mesh bytes requires a 'path' value with a .glb, .ply, or .stl extension "
+ "to infer the mesh file type."
+ )
+ return trimesh.load(BytesIO(bytes_), file_type=file_type)
+
+ def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]:
+ """If in the decodable state, return the feature itself, otherwise flatten the feature into a dictionary."""
+ from .features import Value
+
+ return (
+ self
+ if self.decode
+ else {
+ "bytes": Value("binary"),
+ "path": Value("string"),
+ }
+ )
+
+ def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray]) -> pa.StructArray:
+ """Cast an Arrow array to the Mesh arrow storage type.
+ The Arrow types that can be converted to the Mesh pyarrow storage type are:
+
+ - `pa.string()` - it must contain the "path" data
+ - `pa.large_string()` - it must contain the "path" data (will be cast to string if possible)
+ - `pa.binary()` - it must contain the mesh bytes
+ - `pa.struct({"bytes": pa.binary()})`
+ - `pa.struct({"path": pa.string()})`
+ - `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter
+
+ Args:
+ storage (`Union[pa.StringArray, pa.StructArray]`):
+ PyArrow array to cast.
+
+ Returns:
+ `pa.StructArray`: Array in the Mesh arrow storage type, that is
+ `pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
+ """
+ if pa.types.is_large_string(storage.type):
+ try:
+ storage = storage.cast(pa.string())
+ except pa.ArrowInvalid as e:
+ raise ValueError(
+ f"Failed to cast large_string to string for Mesh feature. "
+ f"This can happen if string values exceed 2GB. "
+ f"Original error: {e}"
+ ) from e
+ if pa.types.is_string(storage.type):
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
+ storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_large_binary(storage.type):
+ storage = array_cast(
+ storage, pa.binary()
+ ) # this can fail in case of big meshes, paths should be used instead
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_binary(storage.type):
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_struct(storage.type):
+ if storage.type.get_field_index("bytes") >= 0:
+ bytes_array = storage.field("bytes")
+ else:
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
+ if storage.type.get_field_index("path") >= 0:
+ path_array = storage.field("path")
+ else:
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null())
+
+ return array_cast(storage, self.pa_type)
+
+ def embed_storage(
+ self, storage: pa.StructArray, token_per_repo_id=None, local_files: bool = True, remote_files: bool = True
+ ) -> pa.StructArray:
+ """Embed mesh files into the Arrow array.
+
+ Args:
+ storage (`pa.StructArray`):
+ PyArrow array to embed.
+ token_per_repo_id (`dict`, optional):
+ Dictionary repo_id -> token to fetch the files bytes.
+ local_files (`bool`, defaults to `True`):
+ Whether to embed local files data in the array.
+ remote_files (`bool`, defaults to `True`):
+ Whether to embed remote files data in the array.
+
+ Returns:
+ `pa.StructArray`: Array in the Mesh arrow storage type, that is
+ `pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
+ """
+ if token_per_repo_id is None:
+ token_per_repo_id = {}
+
+ def path_to_bytes(path):
+ if path is None:
+ return None
+ source_url = path.split("::")[-1]
+ pattern = (
+ config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL
+ )
+ source_url_fields = string_to_dict(source_url, pattern)
+ token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None
+ download_config = DownloadConfig(token=token)
+ with xopen(path, "rb", download_config=download_config) as f:
+ return f.read()
+
+ bytes_array = pa.array(
+ [
+ (
+ path_to_bytes(x["path"])
+ if x["bytes"] is None
+ and ((local_files and is_local_path(x["path"])) or (remote_files and is_remote_url(x["path"])))
+ else x["bytes"]
+ )
+ if x is not None
+ else None
+ for x in storage.to_pylist()
+ ],
+ type=pa.binary(),
+ )
+ path_array = pa.array(
+ [
+ (
+ os.path.basename(path)
+ if (local_files and is_local_path(path)) or (remote_files and is_remote_url(path))
+ else path
+ )
+ if path is not None
+ else None
+ for path in storage.field("path").to_pylist()
+ ],
+ type=pa.string(),
+ )
+ storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null())
+ return array_cast(storage, self.pa_type)
+
+
+def _infer_mesh_file_type(path: Optional[str]) -> Optional[str]:
+ supported_file_types = {"glb", "ply", "stl"}
+ if path is None:
+ return None
+ path_without_archive = path.split("::", 1)[0]
+ path_without_query = path_without_archive.split("?", 1)[0]
+ extension = os.path.splitext(path_without_query)[1].lower().lstrip(".")
+ return extension if extension in supported_file_types else None
+
+
+def encode_trimesh_mesh(mesh: Union["trimesh.Trimesh", "trimesh.Scene"]) -> dict[str, Optional[bytes | str]]:
+ """Encode a trimesh mesh or scene object into GLB bytes."""
+ metadata = getattr(mesh, "metadata", None) or {}
+ path = metadata.get("file_path") or metadata.get("file_name") if isinstance(metadata, dict) else None
+ if path is not None and os.path.isfile(path):
+ return {"path": path, "bytes": None}
+ bytes_ = mesh.export(file_type="glb")
+ return {"path": os.path.basename(path) if path else "mesh.glb", "bytes": bytes_}
diff --git a/venv/Lib/site-packages/datasets/features/nifti.py b/venv/Lib/site-packages/datasets/features/nifti.py
new file mode 100644
index 0000000000000000000000000000000000000000..1e78cad9e28a10aea73e4de15807da3d75672e6e
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/features/nifti.py
@@ -0,0 +1,351 @@
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union
+
+import pyarrow as pa
+
+from .. import config
+from ..download.download_config import DownloadConfig
+from ..table import array_cast
+from ..utils.file_utils import is_local_path, is_remote_url, xopen
+from ..utils.py_utils import no_op_if_value_is_null, string_to_dict
+
+
+if TYPE_CHECKING:
+ import nibabel as nib
+
+ from .features import FeatureType
+
+if config.NIBABEL_AVAILABLE:
+ import nibabel as nib
+
+ class Nifti1ImageWrapper(nib.nifti1.Nifti1Image):
+ """
+ A wrapper around nibabel's Nifti1Image to customize its representation.
+ """
+
+ def __init__(self, nifti_image: nib.nifti1.Nifti1Image):
+ super().__init__(
+ dataobj=nifti_image.dataobj,
+ affine=nifti_image.affine,
+ header=nifti_image.header,
+ extra=nifti_image.extra,
+ file_map=nifti_image.file_map,
+ dtype=nifti_image.get_data_dtype(),
+ )
+ self.nifti_image = nifti_image
+
+ def _repr_html_(self):
+ from ipyniivue import NiiVue, ShowRender, SliceType, Volume
+ from IPython.display import display
+
+ bytes_ = self.nifti_image.to_bytes()
+ nv = NiiVue()
+ nv.set_slice_type(SliceType.MULTIPLANAR)
+ nv.opts.multiplanar_show_render = ShowRender.ALWAYS
+ nv.opts.show_3d_crosshair = True
+ nv.opts.multiplanar_force_render = True
+ name = None
+ if hasattr(self.nifti_image, "file_map"):
+ if (
+ "image" in self.nifti_image.file_map
+ and getattr(self.nifti_image.file_map["image"], "filename", None) is not None
+ ):
+ name = self.nifti_image.file_map["image"].filename
+ if name is None:
+ name = "volume.nii.gz"
+ volume = Volume(name=name, data=bytes_)
+ nv.load_volumes([volume])
+ display(nv)
+
+
+@dataclass
+class Nifti:
+ """
+ **Experimental.**
+ Nifti [`Feature`] to read NIfTI neuroimaging files.
+
+ Input: The Nifti feature accepts as input:
+ - A `str`: Absolute path to the NIfTI file (i.e. random access is allowed).
+ - A `pathlib.Path`: path to the NIfTI file (i.e. random access is allowed).
+ - A `dict` with the keys:
+ - `path`: String with relative path of the NIfTI file in a dataset repository.
+ - `bytes`: Bytes of the NIfTI file.
+ This is useful for archived files with sequential access.
+
+ - A `nibabel` image object (e.g., `nibabel.nifti1.Nifti1Image`).
+
+ Args:
+ decode (`bool`, defaults to `True`):
+ Whether to decode the NIfTI data. If `False` a string with the bytes is returned. `decode=False` is not supported when decoding examples.
+
+ Examples:
+
+ ```py
+ >>> from datasets import Dataset, Nifti
+ >>> ds = Dataset.from_dict({"nifti": ["path/to/file.nii.gz"]}).cast_column("nifti", Nifti())
+ >>> ds.features["nifti"]
+ Nifti(decode=True, id=None)
+ >>> ds[0]["nifti"]
+
+ >>> ds = ds.cast_column("nifti", Nifti(decode=False))
+ >>> ds[0]["nifti"]
+ {'bytes': None,
+ 'path': 'path/to/file.nii.gz'}
+ ```
+ """
+
+ decode: bool = True
+ id: Optional[str] = field(default=None, repr=False)
+
+ # Automatically constructed
+ dtype: ClassVar[str] = "nibabel.nifti1.Nifti1Image"
+ pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()})
+ _type: str = field(default="Nifti", init=False, repr=False)
+
+ def __call__(self):
+ return self.pa_type
+
+ def encode_example(self, value: Union[str, bytes, bytearray, dict, "nib.Nifti1Image"]) -> dict:
+ """Encode example into a format for Arrow.
+
+ Args:
+ value (`str`, `bytes`, `nibabel.Nifti1Image` or `dict`):
+ Data passed as input to Nifti feature.
+
+ Returns:
+ `dict` with "path" and "bytes" fields
+ """
+ if config.NIBABEL_AVAILABLE:
+ import nibabel as nib
+ else:
+ nib = None
+
+ if isinstance(value, str):
+ return {"path": value, "bytes": None}
+ elif isinstance(value, Path):
+ return {"path": str(value.absolute()), "bytes": None}
+ elif isinstance(value, (bytes, bytearray)):
+ return {"path": None, "bytes": value}
+ elif nib is not None and isinstance(value, nib.spatialimages.SpatialImage):
+ # nibabel image object - try to get path or convert to bytes
+ return encode_nibabel_image(value)
+ elif isinstance(value, dict):
+ if value.get("path") is not None and os.path.isfile(value["path"]):
+ # we set "bytes": None to not duplicate the data if they're already available locally
+ return {"bytes": None, "path": value.get("path")}
+ elif value.get("bytes") is not None or value.get("path") is not None:
+ # store the nifti bytes, and path is used to infer the format using the file extension
+ return {"bytes": value.get("bytes"), "path": value.get("path")}
+ else:
+ raise ValueError(
+ f"A nifti sample should have one of 'path' or 'bytes' but they are missing or None in {value}."
+ )
+ else:
+ raise ValueError(
+ f"A nifti sample should be a string, bytes, Path, nibabel image, or dict, but got {type(value)}."
+ )
+
+ def decode_example(self, value: dict, token_per_repo_id=None) -> "Nifti1ImageWrapper":
+ """Decode example NIfTI file into nibabel image object.
+
+ Args:
+ value (`str` or `dict`):
+ A string with the absolute NIfTI file path, a dictionary with
+ keys:
+
+ - `path`: String with absolute or relative NIfTI file path.
+ - `bytes`: The bytes of the NIfTI file.
+
+ token_per_repo_id (`dict`, *optional*):
+ To access and decode NIfTI files from private repositories on
+ the Hub, you can pass a dictionary
+ repo_id (`str`) -> token (`bool` or `str`).
+
+ Returns:
+ `nibabel.Nifti1Image` objects
+ """
+ if config.NIBABEL_AVAILABLE:
+ import nibabel as nib
+ else:
+ raise ImportError("To support decoding NIfTI files, please install 'nibabel'.")
+
+ if token_per_repo_id is None:
+ token_per_repo_id = {}
+
+ path, bytes_ = value["path"], value["bytes"]
+ if bytes_ is None:
+ if path is None:
+ raise ValueError(f"A nifti should have one of 'path' or 'bytes' but both are None in {value}.")
+ else:
+ # gzipped files have the structure: 'gzip://T1.nii::'
+ if path.startswith("gzip://") and is_local_path(path.split("::")[-1]):
+ path = path.split("::")[-1]
+ if is_local_path(path):
+ nifti = nib.load(path)
+ else:
+ source_url = path.split("::")[-1]
+ pattern = (
+ config.HUB_DATASETS_URL
+ if source_url.startswith(config.HF_ENDPOINT)
+ else config.HUB_DATASETS_HFFS_URL
+ )
+ source_url_fields = string_to_dict(source_url, pattern)
+ token = (
+ token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None
+ )
+ download_config = DownloadConfig(token=token)
+ with xopen(path, "rb", download_config=download_config) as f:
+ nifti = nib.load(f)
+ else:
+ import gzip
+
+ if (
+ bytes_[:2] == b"\x1f\x8b"
+ ): # gzip magic number, see https://stackoverflow.com/a/76055284/9534390 or "Magic number" on https://en.wikipedia.org/wiki/Gzip
+ bytes_ = gzip.decompress(bytes_)
+
+ nifti = nib.Nifti1Image.from_bytes(bytes_)
+
+ return Nifti1ImageWrapper(nifti)
+
+ def embed_storage(
+ self, storage: pa.StructArray, token_per_repo_id=None, local_files: bool = True, remote_files: bool = True
+ ) -> pa.StructArray:
+ """Embed NifTI files into the Arrow array.
+
+ Args:
+ storage (`pa.StructArray`):
+ PyArrow array to embed.
+ token_per_repo_id (`dict`, optional):
+ Dictionary repo_id -> token to fetch the files bytes.
+ local_files (`bool`, defaults to `True`)
+ Whether to embed local files data in the array
+
+
+ remote_files (`bool`, defaults to `True`)
+ Whether to embed remote files data in the array.
+ E.g. files with paths that start with hf:// or https://
+
+
+
+ Returns:
+ `pa.StructArray`: Array in the NifTI arrow storage type, that is
+ `pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
+ """
+ if token_per_repo_id is None:
+ token_per_repo_id = {}
+
+ @no_op_if_value_is_null
+ def path_to_bytes(path):
+ source_url = path.split("::")[-1]
+ pattern = (
+ config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL
+ )
+ source_url_fields = string_to_dict(source_url, pattern)
+ token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None
+ download_config = DownloadConfig(token=token)
+ with xopen(path, "rb", download_config=download_config) as f:
+ return f.read()
+
+ bytes_array = pa.array(
+ [
+ (
+ path_to_bytes(x["path"])
+ if x["bytes"] is None
+ and ((local_files and is_local_path(x["path"])) or (remote_files and is_remote_url(x["path"])))
+ else x["bytes"]
+ )
+ if x is not None
+ else None
+ for x in storage.to_pylist()
+ ],
+ type=pa.binary(),
+ )
+ path_array = pa.array(
+ [
+ (
+ os.path.basename(path)
+ if (local_files and is_local_path(path)) or (remote_files and is_remote_url(path))
+ else path
+ )
+ if path is not None
+ else None
+ for path in storage.field("path").to_pylist()
+ ],
+ type=pa.string(),
+ )
+ storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null())
+ return array_cast(storage, self.pa_type)
+
+ def flatten(self) -> Union["FeatureType", Dict[str, "FeatureType"]]:
+ """If in the decodable state, return the feature itself, otherwise flatten the feature into a dictionary."""
+ from .features import Value
+
+ return (
+ self
+ if self.decode
+ else {
+ "bytes": Value("binary"),
+ "path": Value("string"),
+ }
+ )
+
+ def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray, pa.BinaryArray]) -> pa.StructArray:
+ """Cast an Arrow array to the Nifti arrow storage type.
+ The Arrow types that can be converted to the Nifti pyarrow storage type are:
+
+ - `pa.string()` - it must contain the "path" data
+ - `pa.binary()` - it must contain the NIfTI bytes
+ - `pa.struct({"bytes": pa.binary()})`
+ - `pa.struct({"path": pa.string()})`
+ - `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter
+
+ Args:
+ storage (`Union[pa.StringArray, pa.StructArray, pa.BinaryArray]`):
+ PyArrow array to cast.
+
+ Returns:
+ `pa.StructArray`: Array in the Nifti arrow storage type, that is
+ `pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
+ """
+ if pa.types.is_string(storage.type):
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
+ storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_binary(storage.type):
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_struct(storage.type):
+ if storage.type.get_field_index("bytes") >= 0:
+ bytes_array = storage.field("bytes")
+ else:
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
+ if storage.type.get_field_index("path") >= 0:
+ path_array = storage.field("path")
+ else:
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null())
+ return array_cast(storage, self.pa_type)
+
+
+def encode_nibabel_image(img: "nib.Nifti1Image", force_bytes: bool = False) -> dict[str, Optional[Union[str, bytes]]]:
+ """
+ Encode a nibabel image object into a dictionary.
+
+ If the image has an associated file path, returns the path. Otherwise, serializes
+ the image content into bytes.
+
+ Args:
+ img: A nibabel image object (e.g., Nifti1Image).
+ force_bytes: If `True`, always serialize to bytes even if a file path exists. Needed to upload bytes properly.
+
+ Returns:
+ dict: A dictionary with "path" or "bytes" field.
+ """
+ if hasattr(img, "file_map") and img.file_map is not None and not force_bytes:
+ filename = img.file_map["image"].filename
+ return {"path": filename, "bytes": None}
+
+ bytes_data = img.to_bytes()
+ return {"path": None, "bytes": bytes_data}
diff --git a/venv/Lib/site-packages/datasets/features/pdf.py b/venv/Lib/site-packages/datasets/features/pdf.py
new file mode 100644
index 0000000000000000000000000000000000000000..74c7560d5ffdd4dda9adf7e4f4808b1a70842894
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/features/pdf.py
@@ -0,0 +1,309 @@
+import os
+from dataclasses import dataclass, field
+from io import BytesIO
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union
+
+import pyarrow as pa
+
+from .. import config
+from ..download.download_config import DownloadConfig
+from ..table import array_cast
+from ..utils.file_utils import is_local_path, is_remote_url, xopen
+from ..utils.py_utils import no_op_if_value_is_null, string_to_dict
+
+
+if TYPE_CHECKING:
+ import pdfplumber
+
+ from .features import FeatureType
+
+
+def pdf_to_bytes(pdf: "pdfplumber.pdf.PDF") -> bytes:
+ """Convert a pdfplumber.pdf.PDF object to bytes."""
+ with BytesIO() as buffer:
+ for page in pdf.pages:
+ buffer.write(page.pdf.stream)
+ return buffer.getvalue()
+
+
+@dataclass
+class Pdf:
+ """
+ **Experimental.**
+ Pdf [`Feature`] to read pdf documents from a pdf file.
+
+ Input: The Pdf feature accepts as input:
+ - A `str`: Absolute path to the pdf file (i.e. random access is allowed).
+ - A `pathlib.Path`: path to the pdf file (i.e. random access is allowed).
+ - A `dict` with the keys:
+ - `path`: String with relative path of the pdf file in a dataset repository.
+ - `bytes`: Bytes of the pdf file.
+ This is useful for archived files with sequential access.
+
+ - A `pdfplumber.pdf.PDF`: pdfplumber pdf object.
+
+ Args:
+ decode (`bool`, defaults to `True`):
+ Whether to decode the pdf data. If `False`,
+ returns the underlying dictionary in the format `{"path": pdf_path, "bytes": pdf_bytes}`.
+
+ Examples:
+
+ ```py
+ >>> from datasets import Dataset, Pdf
+ >>> ds = Dataset.from_dict({"pdf": ["path/to/pdf/file.pdf"]}).cast_column("pdf", Pdf())
+ >>> ds.features["pdf"]
+ Pdf(decode=True, id=None)
+ >>> ds[0]["pdf"]
+
+ >>> ds = ds.cast_column("pdf", Pdf(decode=False))
+ >>> ds[0]["pdf"]
+ {'bytes': None,
+ 'path': 'path/to/pdf/file.pdf'}
+ ```
+ """
+
+ decode: bool = True
+ id: Optional[str] = field(default=None, repr=False)
+
+ # Automatically constructed
+ dtype: ClassVar[str] = "pdfplumber.pdf.PDF"
+ pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()})
+ _type: str = field(default="Pdf", init=False, repr=False)
+
+ def __call__(self):
+ return self.pa_type
+
+ def encode_example(self, value: Union[str, bytes, bytearray, dict, "pdfplumber.pdf.PDF"]) -> dict:
+ """Encode example into a format for Arrow.
+
+ Args:
+ value (`str`, `bytes`, `pdfplumber.pdf.PDF` or `dict`):
+ Data passed as input to Pdf feature.
+
+ Returns:
+ `dict` with "path" and "bytes" fields
+ """
+ if config.PDFPLUMBER_AVAILABLE:
+ import pdfplumber
+ else:
+ pdfplumber = None
+
+ if isinstance(value, str):
+ return {"path": value, "bytes": None}
+ elif isinstance(value, Path):
+ return {"path": str(value.absolute()), "bytes": None}
+ elif isinstance(value, (bytes, bytearray)):
+ return {"path": None, "bytes": value}
+ elif pdfplumber is not None and isinstance(value, pdfplumber.pdf.PDF):
+ # convert the pdfplumber.pdf.PDF to bytes
+ return encode_pdfplumber_pdf(value)
+ elif value.get("path") is not None and os.path.isfile(value["path"]):
+ # we set "bytes": None to not duplicate the data if they're already available locally
+ return {"bytes": None, "path": value.get("path")}
+ elif value.get("bytes") is not None or value.get("path") is not None:
+ # store the pdf bytes, and path is used to infer the pdf format using the file extension
+ return {"bytes": value.get("bytes"), "path": value.get("path")}
+ else:
+ raise ValueError(
+ f"A pdf sample should have one of 'path' or 'bytes' but they are missing or None in {value}."
+ )
+
+ def decode_example(self, value: dict, token_per_repo_id=None) -> "pdfplumber.pdf.PDF":
+ """Decode example pdf file into pdf data.
+
+ Args:
+ value (`str` or `dict`):
+ A string with the absolute pdf file path, a dictionary with
+ keys:
+
+ - `path`: String with absolute or relative pdf file path.
+ - `bytes`: The bytes of the pdf file.
+
+ token_per_repo_id (`dict`, *optional*):
+ To access and decode pdf files from private repositories on
+ the Hub, you can pass a dictionary
+ repo_id (`str`) -> token (`bool` or `str`).
+
+ Returns:
+ `pdfplumber.pdf.PDF`
+ """
+ if not self.decode:
+ raise RuntimeError("Decoding is disabled for this feature. Please use Pdf(decode=True) instead.")
+
+ if config.PDFPLUMBER_AVAILABLE:
+ import pdfplumber
+ else:
+ raise ImportError("To support decoding pdfs, please install 'pdfplumber'.")
+
+ if token_per_repo_id is None:
+ token_per_repo_id = {}
+
+ path, bytes_ = value["path"], value["bytes"]
+ if bytes_ is None:
+ if path is None:
+ raise ValueError(f"A pdf should have one of 'path' or 'bytes' but both are None in {value}.")
+ else:
+ if is_local_path(path):
+ pdf = pdfplumber.open(path)
+ else:
+ source_url = path.split("::")[-1]
+ pattern = (
+ config.HUB_DATASETS_URL
+ if source_url.startswith(config.HF_ENDPOINT)
+ else config.HUB_DATASETS_HFFS_URL
+ )
+ try:
+ repo_id = string_to_dict(source_url, pattern)["repo_id"]
+ token = token_per_repo_id.get(repo_id)
+ except ValueError:
+ token = None
+ download_config = DownloadConfig(token=token)
+ f = xopen(path, "rb", download_config=download_config)
+ return pdfplumber.open(f)
+ else:
+ with pdfplumber.open(BytesIO(bytes_)) as p:
+ pdf = p
+
+ return pdf
+
+ def flatten(self) -> Union["FeatureType", Dict[str, "FeatureType"]]:
+ """If in the decodable state, return the feature itself, otherwise flatten the feature into a dictionary."""
+ from .features import Value
+
+ return (
+ self
+ if self.decode
+ else {
+ "bytes": Value("binary"),
+ "path": Value("string"),
+ }
+ )
+
+ def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray, pa.ListArray]) -> pa.StructArray:
+ """Cast an Arrow array to the Pdf arrow storage type.
+ The Arrow types that can be converted to the Pdf pyarrow storage type are:
+
+ - `pa.string()` - it must contain the "path" data
+ - `pa.binary()` - it must contain the image bytes
+ - `pa.struct({"bytes": pa.binary()})`
+ - `pa.struct({"path": pa.string()})`
+ - `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter
+ - `pa.list(*)` - it must contain the pdf array data
+
+ Args:
+ storage (`Union[pa.StringArray, pa.StructArray, pa.ListArray]`):
+ PyArrow array to cast.
+
+ Returns:
+ `pa.StructArray`: Array in the Pdf arrow storage type, that is
+ `pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
+ """
+ if pa.types.is_string(storage.type):
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
+ storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_binary(storage.type):
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_struct(storage.type):
+ if storage.type.get_field_index("bytes") >= 0:
+ bytes_array = storage.field("bytes")
+ else:
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
+ if storage.type.get_field_index("path") >= 0:
+ path_array = storage.field("path")
+ else:
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null())
+ return array_cast(storage, self.pa_type)
+
+ def embed_storage(
+ self, storage: pa.StructArray, token_per_repo_id=None, local_files: bool = True, remote_files: bool = True
+ ) -> pa.StructArray:
+ """Embed PDF files into the Arrow array.
+
+ Args:
+ storage (`pa.StructArray`):
+ PyArrow array to embed.
+ token_per_repo_id (`dict`, optional):
+ Dictionary repo_id -> token to fetch the files bytes.
+ local_files (`bool`, defaults to `True`)
+ Whether to embed local files data in the array
+
+
+ remote_files (`bool`, defaults to `True`)
+ Whether to embed remote files data in the array.
+ E.g. files with paths that start with hf:// or https://
+
+
+
+ Returns:
+ `pa.StructArray`: Array in the PDF arrow storage type, that is
+ `pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
+ """
+ if token_per_repo_id is None:
+ token_per_repo_id = {}
+
+ @no_op_if_value_is_null
+ def path_to_bytes(path):
+ source_url = path.split("::")[-1]
+ pattern = (
+ config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL
+ )
+ source_url_fields = string_to_dict(source_url, pattern)
+ token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None
+ download_config = DownloadConfig(token=token)
+ with xopen(path, "rb", download_config=download_config) as f:
+ return f.read()
+
+ bytes_array = pa.array(
+ [
+ (
+ path_to_bytes(x["path"])
+ if x["bytes"] is None
+ and ((local_files and is_local_path(x["path"])) or (remote_files and is_remote_url(x["path"])))
+ else x["bytes"]
+ )
+ if x is not None
+ else None
+ for x in storage.to_pylist()
+ ],
+ type=pa.binary(),
+ )
+ path_array = pa.array(
+ [
+ (
+ os.path.basename(path)
+ if (local_files and is_local_path(path)) or (remote_files and is_remote_url(path))
+ else path
+ )
+ if path is not None
+ else None
+ for path in storage.field("path").to_pylist()
+ ],
+ type=pa.string(),
+ )
+ storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null())
+ return array_cast(storage, self.pa_type)
+
+
+def encode_pdfplumber_pdf(pdf: "pdfplumber.pdf.PDF") -> dict:
+ """
+ Encode a pdfplumber.pdf.PDF object into a dictionary.
+
+ If the PDF has an associated file path, returns the path. Otherwise, serializes
+ the PDF content into bytes.
+
+ Args:
+ pdf (pdfplumber.pdf.PDF): A pdfplumber PDF object.
+
+ Returns:
+ dict: A dictionary with "path" or "bytes" field.
+ """
+ if hasattr(pdf, "stream") and hasattr(pdf.stream, "name") and pdf.stream.name:
+ # Return the path if the PDF has an associated file path
+ return {"path": pdf.stream.name, "bytes": None}
+ else:
+ # Convert the PDF to bytes if no path is available
+ return {"path": None, "bytes": pdf_to_bytes(pdf)}
diff --git a/venv/Lib/site-packages/datasets/features/translation.py b/venv/Lib/site-packages/datasets/features/translation.py
new file mode 100644
index 0000000000000000000000000000000000000000..9bee3aa2c11f26edbe0d17138a71fde96e3d3541
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/features/translation.py
@@ -0,0 +1,129 @@
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union
+
+import pyarrow as pa
+
+
+if TYPE_CHECKING:
+ from .features import FeatureType
+
+
+@dataclass
+class Translation:
+ """`Feature` for translations with fixed languages per example.
+ Here for compatibility with tfds.
+
+ Args:
+ languages (`dict`):
+ A dictionary for each example mapping string language codes to string translations.
+
+ Example:
+
+ ```python
+ >>> # At construction time:
+ >>> datasets.features.Translation(languages=['en', 'fr', 'de'])
+ >>> # During data generation:
+ >>> yield {
+ ... 'en': 'the cat',
+ ... 'fr': 'le chat',
+ ... 'de': 'die katze'
+ ... }
+ ```
+ """
+
+ languages: list[str]
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ dtype: ClassVar[str] = "dict"
+ pa_type: ClassVar[Any] = None
+ _type: str = field(default="Translation", init=False, repr=False)
+
+ def __call__(self):
+ return pa.struct({lang: pa.string() for lang in sorted(self.languages)})
+
+ def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]:
+ """Flatten the Translation feature into a dictionary."""
+ from .features import Value
+
+ return {k: Value("string") for k in sorted(self.languages)}
+
+
+@dataclass
+class TranslationVariableLanguages:
+ """`Feature` for translations with variable languages per example.
+ Here for compatibility with tfds.
+
+ Args:
+ languages (`dict`):
+ A dictionary for each example mapping string language codes to one or more string translations.
+ The languages present may vary from example to example.
+
+ Returns:
+ - `language` or `translation` (variable-length 1D `tf.Tensor` of `tf.string`):
+ Language codes sorted in ascending order or plain text translations, sorted to align with language codes.
+
+ Example:
+
+ ```python
+ >>> # At construction time:
+ >>> datasets.features.TranslationVariableLanguages(languages=['en', 'fr', 'de'])
+ >>> # During data generation:
+ >>> yield {
+ ... 'en': 'the cat',
+ ... 'fr': ['le chat', 'la chatte,']
+ ... 'de': 'die katze'
+ ... }
+ >>> # Tensor returned :
+ >>> {
+ ... 'language': ['en', 'de', 'fr', 'fr'],
+ ... 'translation': ['the cat', 'die katze', 'la chatte', 'le chat'],
+ ... }
+ ```
+ """
+
+ languages: Optional[list] = None
+ num_languages: Optional[int] = None
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ dtype: ClassVar[str] = "dict"
+ pa_type: ClassVar[Any] = None
+ _type: str = field(default="TranslationVariableLanguages", init=False, repr=False)
+
+ def __post_init__(self):
+ self.languages = sorted(set(self.languages)) if self.languages else None
+ self.num_languages = len(self.languages) if self.languages else None
+
+ def __call__(self):
+ return pa.struct({"language": pa.list_(pa.string()), "translation": pa.list_(pa.string())})
+
+ def encode_example(self, translation_dict):
+ lang_set = set(self.languages)
+ if set(translation_dict) == {"language", "translation"}:
+ return translation_dict
+ elif self.languages and set(translation_dict) - lang_set:
+ raise ValueError(
+ f"Some languages in example ({', '.join(sorted(set(translation_dict) - lang_set))}) are not in valid set ({', '.join(lang_set)})."
+ )
+
+ # Convert dictionary into tuples, splitting out cases where there are
+ # multiple translations for a single language.
+ translation_tuples = []
+ for lang, text in translation_dict.items():
+ if isinstance(text, str):
+ translation_tuples.append((lang, text))
+ else:
+ translation_tuples.extend([(lang, el) for el in text])
+
+ # Ensure translations are in ascending order by language code.
+ languages, translations = zip(*sorted(translation_tuples))
+
+ return {"language": languages, "translation": translations}
+
+ def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]:
+ """Flatten the TranslationVariableLanguages feature into a dictionary."""
+ from .features import List, Value
+
+ return {
+ "language": List(Value("string")),
+ "translation": List(Value("string")),
+ }
diff --git a/venv/Lib/site-packages/datasets/features/video.py b/venv/Lib/site-packages/datasets/features/video.py
new file mode 100644
index 0000000000000000000000000000000000000000..8522c2cb1225aff2f8bb21a28a8a22570c8650bd
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/features/video.py
@@ -0,0 +1,415 @@
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, ClassVar, Literal, Optional, TypedDict, Union
+
+import numpy as np
+import pyarrow as pa
+
+from .. import config
+from ..download.download_config import DownloadConfig
+from ..table import array_cast
+from ..utils.file_utils import is_local_path, is_remote_url, xopen
+from ..utils.py_utils import no_op_if_value_is_null, string_to_dict
+
+
+if TYPE_CHECKING:
+ import torch
+ from torchcodec.decoders import VideoDecoder
+
+ from .features import FeatureType
+
+
+class Example(TypedDict):
+ path: Optional[str]
+ bytes: Optional[bytes]
+
+
+@dataclass
+class Video:
+ """
+ Video [`Feature`] to read video data from a video file.
+
+ Input: The Video feature accepts as input:
+ - A `str`: Absolute path to the video file (i.e. random access is allowed).
+ - A `pathlib.Path`: path to the video file (i.e. random access is allowed).
+ - A `dict` with the keys:
+
+ - `path`: String with relative path of the video file in a dataset repository.
+ - `bytes`: Bytes of the video file.
+
+ This is useful for parquet or webdataset files which embed video files.
+
+ - A `torchcodec.decoders.VideoDecoder`: torchcodec video decoder object.
+
+ Output: The Video features output data as `torchcodec.decoders.VideoDecoder` objects.
+
+ Args:
+ decode (`bool`, defaults to `True`):
+ Whether to decode the video data. If `False`,
+ returns the underlying dictionary in the format `{"path": video_path, "bytes": video_bytes}`.
+ stream_index (`int`, *optional*):
+ The streaming index to use from the file. If `None` defaults to the "best" index.
+ dimension_order (`str`, defaults to `NCHW`):
+ The dimension order of the decoded frames.
+ where N is the batch size, C is the number of channels,
+ H is the height, and W is the width of the frames.
+ num_ffmpeg_threads (`int`, defaults to `1`):
+ The number of threads to use for decoding the video. (Recommended to keep this at 1)
+ device (`str` or `torch.device`, defaults to `cpu`):
+ The device to use for decoding the video.
+ seek_mode (`str`, defaults to `exact`):
+ Determines if frame access will be “exact” or “approximate”.
+ Exact guarantees that requesting frame i will always return frame i, but doing so requires an initial scan of the file.
+ Approximate is faster as it avoids scanning the file, but less accurate as it uses the file's metadata to calculate where i probably is.
+ read more [here](https://docs.pytorch.org/torchcodec/stable/generated_examples/approximate_mode.html#sphx-glr-generated-examples-approximate-mode-py)
+
+ Examples:
+
+ ```py
+ >>> from datasets import Dataset, Video
+ >>> ds = Dataset.from_dict({"video":["path/to/Screen Recording.mov"]}).cast_column("video", Video())
+ >>> ds.features["video"]
+ Video(decode=True, id=None)
+ >>> ds[0]["video"]
+
+ >>> video = ds[0]["video"]
+ >>> video.get_frames_in_range(0, 10)
+ FrameBatch:
+ data (shape): torch.Size([10, 3, 50, 66])
+ pts_seconds: tensor([0.4333, 0.4333, 0.4333, 0.4333, 0.4333, 0.4333, 0.4333, 0.4333, 0.4333,
+ 0.4333], dtype=torch.float64)
+ duration_seconds: tensor([0.0167, 0.0167, 0.0167, 0.0167, 0.0167, 0.0167, 0.0167, 0.0167, 0.0167,
+ 0.0167], dtype=torch.float64)
+ >>> ds.cast_column('video', Video(decode=False))[0]["video]
+ {'bytes': None,
+ 'path': 'path/to/Screen Recording.mov'}
+ ```
+ """
+
+ decode: bool = True
+ stream_index: Optional[int] = None
+ dimension_order: Literal["NCHW", "NHWC"] = "NCHW"
+ num_ffmpeg_threads: int = 1
+ device: Optional[Union[str, "torch.device"]] = "cpu"
+ seek_mode: Literal["exact", "approximate"] = "exact"
+ id: Optional[str] = field(default=None, repr=False)
+ # Automatically constructed
+ dtype: ClassVar[str] = "torchcodec.decoders.VideoDecoder"
+ pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()})
+ _type: str = field(default="Video", init=False, repr=False)
+
+ def __call__(self):
+ return self.pa_type
+
+ def encode_example(self, value: Union[str, bytes, bytearray, Example, np.ndarray, "VideoDecoder"]) -> Example:
+ """Encode example into a format for Arrow.
+
+ Args:
+ value (`str`, `np.ndarray`, `bytes`, `bytearray`, `VideoDecoder` or `dict`):
+ Data passed as input to Video feature.
+
+ Returns:
+ `dict` with "path" and "bytes" fields
+ """
+ if value is None:
+ raise ValueError("value must be provided")
+
+ if config.TORCHCODEC_AVAILABLE:
+ from torchcodec.decoders import VideoDecoder
+ else:
+ VideoDecoder = None
+
+ if isinstance(value, list):
+ value = np.array(value)
+
+ if isinstance(value, str):
+ return {"path": value, "bytes": None}
+ elif isinstance(value, Path):
+ return {"path": str(value.absolute()), "bytes": None}
+ elif isinstance(value, (bytes, bytearray)):
+ return {"path": None, "bytes": value}
+ elif isinstance(value, np.ndarray):
+ # convert the video array to bytes
+ return encode_np_array(value)
+ elif VideoDecoder is not None and isinstance(value, VideoDecoder):
+ # convert the torchcodec video decoder to bytes
+ return encode_torchcodec_video(value)
+ elif isinstance(value, dict):
+ path, bytes_ = value.get("path"), value.get("bytes")
+ if path is not None and os.path.isfile(path):
+ # we set "bytes": None to not duplicate the data if they're already available locally
+ return {"bytes": None, "path": path}
+ elif bytes_ is not None or path is not None:
+ # store the video bytes, and path is used to infer the video format using the file extension
+ return {"bytes": bytes_, "path": path}
+ else:
+ raise ValueError(
+ f"A video sample should have one of 'path' or 'bytes' but they are missing or None in {value}."
+ )
+ else:
+ raise TypeError(f"Unsupported encode_example type: {type(value)}")
+
+ def decode_example(
+ self,
+ value: Union[str, Example],
+ token_per_repo_id: Optional[dict[str, Union[bool, str]]] = None,
+ ) -> "VideoDecoder":
+ """Decode example video file into video data.
+
+ Args:
+ value (`str` or `dict`):
+ A string with the absolute video file path, a dictionary with
+ keys:
+
+ - `path`: String with absolute or relative video file path.
+ - `bytes`: The bytes of the video file.
+ token_per_repo_id (`dict`, *optional*):
+ To access and decode
+ video files from private repositories on the Hub, you can pass
+ a dictionary repo_id (`str`) -> token (`bool` or `str`).
+
+ Returns:
+ `torchcodec.decoders.VideoDecoder`
+ """
+ if not self.decode:
+ raise RuntimeError("Decoding is disabled for this feature. Please use Video(decode=True) instead.")
+
+ if config.TORCHCODEC_AVAILABLE:
+ from torchcodec.decoders import VideoDecoder
+
+ else:
+ raise ImportError("To support decoding videos, please install 'torchcodec'.")
+
+ if token_per_repo_id is None:
+ token_per_repo_id = {}
+
+ if isinstance(value, str):
+ path, bytes_ = value, None
+ else:
+ path, bytes_ = value["path"], value["bytes"]
+
+ if bytes_ is None:
+ if path is None:
+ raise ValueError(f"A video should have one of 'path' or 'bytes' but both are None in {value}.")
+ elif is_local_path(path):
+ video = VideoDecoder(
+ path,
+ stream_index=self.stream_index,
+ dimension_order=self.dimension_order,
+ num_ffmpeg_threads=self.num_ffmpeg_threads,
+ device=self.device,
+ seek_mode=self.seek_mode,
+ )
+ else:
+ video = hf_video_reader(
+ path,
+ token_per_repo_id=token_per_repo_id,
+ dimension_order=self.dimension_order,
+ num_ffmpeg_threads=self.num_ffmpeg_threads,
+ device=self.device,
+ seek_mode=self.seek_mode,
+ )
+ else:
+ video = VideoDecoder(
+ bytes_,
+ stream_index=self.stream_index,
+ dimension_order=self.dimension_order,
+ num_ffmpeg_threads=self.num_ffmpeg_threads,
+ device=self.device,
+ seek_mode=self.seek_mode,
+ )
+ video._hf_encoded = {"path": path, "bytes": bytes_}
+ video.metadata.path = path
+ return video
+
+ def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]:
+ """If in the decodable state, return the feature itself, otherwise flatten the feature into a dictionary."""
+ from .features import Value
+
+ return (
+ self
+ if self.decode
+ else {
+ "bytes": Value("binary"),
+ "path": Value("string"),
+ }
+ )
+
+ def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray, pa.ListArray]) -> pa.StructArray:
+ """Cast an Arrow array to the Video arrow storage type.
+ The Arrow types that can be converted to the Video pyarrow storage type are:
+
+ - `pa.string()` - it must contain the "path" data
+ - `pa.binary()` - it must contain the video bytes
+ - `pa.struct({"bytes": pa.binary()})`
+ - `pa.struct({"path": pa.string()})`
+ - `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter
+ - `pa.list(*)` - it must contain the video array data
+
+ Args:
+ storage (`Union[pa.StringArray, pa.StructArray, pa.ListArray]`):
+ PyArrow array to cast.
+
+ Returns:
+ `pa.StructArray`: Array in the Video arrow storage type, that is
+ `pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
+ """
+ if pa.types.is_string(storage.type):
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
+ storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_large_binary(storage.type):
+ storage = array_cast(
+ storage, pa.binary()
+ ) # this can fail in case of big videos, paths should be used instead
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_binary(storage.type):
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_struct(storage.type):
+ if storage.type.get_field_index("bytes") >= 0:
+ bytes_array = storage.field("bytes")
+ else:
+ bytes_array = pa.array([None] * len(storage), type=pa.binary())
+ if storage.type.get_field_index("path") >= 0:
+ path_array = storage.field("path")
+ else:
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null())
+ elif pa.types.is_list(storage.type):
+ bytes_array = pa.array(
+ [encode_np_array(np.array(arr))["bytes"] if arr is not None else None for arr in storage.to_pylist()],
+ type=pa.binary(),
+ )
+ path_array = pa.array([None] * len(storage), type=pa.string())
+ storage = pa.StructArray.from_arrays(
+ [bytes_array, path_array], ["bytes", "path"], mask=bytes_array.is_null()
+ )
+ return array_cast(storage, self.pa_type)
+
+ def embed_storage(
+ self, storage: pa.StructArray, token_per_repo_id=None, local_files: bool = True, remote_files: bool = True
+ ) -> pa.StructArray:
+ """Embed image files into the Arrow array.
+
+ Args:
+ storage (`pa.StructArray`):
+ PyArrow array to embed.
+ token_per_repo_id (`dict`, optional):
+ Dictionary repo_id -> token to fetch the files bytes.
+ local_files (`bool`, defaults to `True`)
+ Whether to embed local files data in the array
+
+
+ remote_files (`bool`, defaults to `True`)
+ Whether to embed remote files data in the array.
+ E.g. files with paths that start with hf:// or https://
+
+
+
+ Returns:
+ `pa.StructArray`: Array in the Video arrow storage type, that is
+ `pa.struct({"bytes": pa.binary(), "path": pa.string()})`.
+ """
+ if token_per_repo_id is None:
+ token_per_repo_id = {}
+
+ @no_op_if_value_is_null
+ def path_to_bytes(path):
+ source_url = path.split("::")[-1]
+ pattern = (
+ config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL
+ )
+ source_url_fields = string_to_dict(source_url, pattern)
+ token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None
+ download_config = DownloadConfig(token=token)
+ with xopen(path, "rb", download_config=download_config) as f:
+ return f.read()
+
+ bytes_array = pa.array(
+ [
+ (
+ path_to_bytes(x["path"])
+ if x["bytes"] is None
+ and ((local_files and is_local_path(x["path"])) or (remote_files and is_remote_url(x["path"])))
+ else x["bytes"]
+ )
+ if x is not None
+ else None
+ for x in storage.to_pylist()
+ ],
+ type=pa.binary(),
+ )
+ path_array = pa.array(
+ [
+ (
+ os.path.basename(path)
+ if (local_files and is_local_path(path)) or (remote_files and is_remote_url(path))
+ else path
+ )
+ if path is not None
+ else None
+ for path in storage.field("path").to_pylist()
+ ],
+ type=pa.string(),
+ )
+ storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null())
+ return array_cast(storage, self.pa_type)
+
+
+def video_to_bytes(video: "VideoDecoder") -> bytes:
+ """Convert a torchcodec Video object to bytes using native compression if possible"""
+ raise NotImplementedError()
+
+
+def encode_torchcodec_video(video: "VideoDecoder") -> Example:
+ if hasattr(video, "_hf_encoded"):
+ return video._hf_encoded
+ else:
+ raise NotImplementedError(
+ "Encoding a VideoDecoder that doesn't come from datasets.Video.decode() is not implemented"
+ )
+
+
+def encode_np_array(array: np.ndarray) -> Example:
+ raise NotImplementedError()
+
+
+# No monkey patch needed!
+# 1. store the encoded video data {"path": ..., "bytes": ...} in `video._hf_encoded``
+# 2. add support for hf:// files
+
+
+def hf_video_reader(
+ path: str,
+ token_per_repo_id: Optional[dict[str, Union[bool, str]]] = None,
+ stream: str = "video",
+ dimension_order: Literal["NCHW", "NHWC"] = "NCHW",
+ num_ffmpeg_threads: int = 1,
+ device: Optional[Union[str, "torch.device"]] = "cpu",
+ seek_mode: Literal["exact", "approximate"] = "exact",
+) -> "VideoDecoder":
+ from torchcodec.decoders import VideoDecoder
+
+ # Load the file from HF
+ if token_per_repo_id is None:
+ token_per_repo_id = {}
+ source_url = path.split("::")[-1]
+ pattern = config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL
+ source_url_fields = string_to_dict(source_url, pattern)
+ token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None
+ download_config = DownloadConfig(token=token)
+ f = xopen(path, "rb", download_config=download_config)
+
+ # Instantiate the VideoDecoder
+ stream_id = 0 if len(stream.split(":")) == 1 else int(stream.split(":")[1])
+ vd = VideoDecoder(
+ f,
+ stream_index=stream_id,
+ dimension_order=dimension_order,
+ num_ffmpeg_threads=num_ffmpeg_threads,
+ device=device,
+ seek_mode=seek_mode,
+ )
+ return vd
diff --git a/venv/Lib/site-packages/datasets/filesystems/__init__.py b/venv/Lib/site-packages/datasets/filesystems/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..d19d58bca1fd8f48e2ef9e5fcdfe17f7738692ac
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/filesystems/__init__.py
@@ -0,0 +1,47 @@
+import importlib
+import shutil
+import warnings
+from typing import List
+
+import fsspec
+import fsspec.asyn
+from fsspec.implementations.local import LocalFileSystem
+
+from . import compression
+
+
+COMPRESSION_FILESYSTEMS: list[compression.BaseCompressedFileFileSystem] = [
+ compression.Bz2FileSystem,
+ compression.GzipFileSystem,
+ compression.Lz4FileSystem,
+ compression.XzFileSystem,
+ compression.ZstdFileSystem,
+]
+
+# Register custom filesystems
+for fs_class in COMPRESSION_FILESYSTEMS:
+ if fs_class.protocol in fsspec.registry and fsspec.registry[fs_class.protocol] is not fs_class:
+ warnings.warn(f"A filesystem protocol was already set for {fs_class.protocol} and will be overwritten.")
+ fsspec.register_implementation(fs_class.protocol, fs_class, clobber=True)
+
+
+def is_remote_filesystem(fs: fsspec.AbstractFileSystem) -> bool:
+ """
+ Checks if `fs` is a remote filesystem.
+
+ Args:
+ fs (`fsspec.spec.AbstractFileSystem`):
+ An abstract super-class for pythonic file-systems, e.g. `fsspec.filesystem(\'file\')` or `s3fs.S3FileSystem`.
+ """
+ return not isinstance(fs, LocalFileSystem)
+
+
+def rename(fs: fsspec.AbstractFileSystem, src: str, dst: str):
+ """
+ Renames the file `src` in `fs` to `dst`.
+ """
+ if not is_remote_filesystem(fs):
+ # LocalFileSystem.mv does copy + rm, it is more efficient to simply move a local directory
+ shutil.move(fs._strip_protocol(src), fs._strip_protocol(dst))
+ else:
+ fs.mv(src, dst, recursive=True)
diff --git a/venv/Lib/site-packages/datasets/filesystems/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/filesystems/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0c0a7333d040ebbd5a97f8c559c701ec34b490e9
Binary files /dev/null and b/venv/Lib/site-packages/datasets/filesystems/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/filesystems/__pycache__/compression.cpython-311.pyc b/venv/Lib/site-packages/datasets/filesystems/__pycache__/compression.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8ec554268b312f21c7caaab1abe0dddbb5fd7d7c
Binary files /dev/null and b/venv/Lib/site-packages/datasets/filesystems/__pycache__/compression.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/filesystems/compression.py b/venv/Lib/site-packages/datasets/filesystems/compression.py
new file mode 100644
index 0000000000000000000000000000000000000000..237b06d75008f8354a4b43d19f6a023e02991393
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/filesystems/compression.py
@@ -0,0 +1,127 @@
+import os
+from functools import partial
+from typing import Optional
+
+import fsspec
+from fsspec.archive import AbstractArchiveFileSystem
+
+
+class BaseCompressedFileFileSystem(AbstractArchiveFileSystem):
+ """Read contents of compressed file as a filesystem with one file inside."""
+
+ root_marker = ""
+ protocol: str = (
+ None # protocol passed in prefix to the url. ex: "gzip", for gzip://file.txt::http://foo.bar/file.txt.gz
+ )
+ compression: str = None # compression type in fsspec. ex: "gzip"
+ extensions: list[str] = None # extensions of the filename to strip. ex: ".gz" to get file.txt from file.txt.gz
+
+ def __init__(
+ self, fo: str = "", target_protocol: Optional[str] = None, target_options: Optional[dict] = None, **kwargs
+ ):
+ """
+ The compressed file system can be instantiated from any compressed file.
+ It reads the contents of compressed file as a filesystem with one file inside, as if it was an archive.
+
+ The single file inside the filesystem is named after the compresssed file,
+ without the compression extension at the end of the filename.
+
+ Args:
+ fo (:obj:``str``): Path to compressed file. Will fetch file using ``fsspec.open()``
+ mode (:obj:``str``): Currently, only 'rb' accepted
+ target_protocol(:obj:``str``, optional): To override the FS protocol inferred from a URL.
+ target_options (:obj:``dict``, optional): Kwargs passed when instantiating the target FS.
+ """
+ super().__init__(self, **kwargs)
+ self.fo = fo.__fspath__() if hasattr(fo, "__fspath__") else fo
+ # always open as "rb" since fsspec can then use the TextIOWrapper to make it work for "r" mode
+ self._open_with_fsspec = partial(
+ fsspec.open,
+ self.fo,
+ mode="rb",
+ protocol=target_protocol,
+ compression=self.compression,
+ client_kwargs={
+ "requote_redirect_url": False, # see https://github.com/huggingface/datasets/pull/5459
+ "trust_env": True, # Enable reading proxy env variables.
+ **(target_options or {}).pop("client_kwargs", {}), # To avoid issues if it was already passed.
+ },
+ **(target_options or {}),
+ )
+ self.compressed_name = os.path.basename(self.fo.split("::")[0])
+ self.uncompressed_name = (
+ self.compressed_name[: self.compressed_name.rindex(".")]
+ if "." in self.compressed_name
+ else self.compressed_name
+ )
+ self.dir_cache = None
+
+ @classmethod
+ def _strip_protocol(cls, path):
+ # compressed file paths are always relative to the archive root
+ return super()._strip_protocol(path).lstrip("/")
+
+ def _get_dirs(self):
+ if self.dir_cache is None:
+ f = {**self._open_with_fsspec().fs.info(self.fo), "name": self.uncompressed_name}
+ self.dir_cache = {f["name"]: f}
+
+ def cat(self, path: str):
+ with self._open_with_fsspec().open() as f:
+ return f.read()
+
+ def _open(
+ self,
+ path: str,
+ mode: str = "rb",
+ block_size=None,
+ autocommit=True,
+ cache_options=None,
+ **kwargs,
+ ):
+ path = self._strip_protocol(path)
+ if mode != "rb":
+ raise ValueError(f"Tried to read with mode {mode} on file {self.fo} opened with mode 'rb'")
+ return self._open_with_fsspec().open()
+
+
+class Bz2FileSystem(BaseCompressedFileFileSystem):
+ """Read contents of BZ2 file as a filesystem with one file inside."""
+
+ protocol = "bz2"
+ compression = "bz2"
+ extensions = [".bz2"]
+
+
+class GzipFileSystem(BaseCompressedFileFileSystem):
+ """Read contents of GZIP file as a filesystem with one file inside."""
+
+ protocol = "gzip"
+ compression = "gzip"
+ extensions = [".gz", ".gzip"]
+
+
+class Lz4FileSystem(BaseCompressedFileFileSystem):
+ """Read contents of LZ4 file as a filesystem with one file inside."""
+
+ protocol = "lz4"
+ compression = "lz4"
+ extensions = [".lz4"]
+
+
+class XzFileSystem(BaseCompressedFileFileSystem):
+ """Read contents of .xz (LZMA) file as a filesystem with one file inside."""
+
+ protocol = "xz"
+ compression = "xz"
+ extensions = [".xz"]
+
+
+class ZstdFileSystem(BaseCompressedFileFileSystem):
+ """
+ Read contents of .zstd file as a filesystem with one file inside.
+ """
+
+ protocol = "zstd"
+ compression = "zstd"
+ extensions = [".zst", ".zstd"]
diff --git a/venv/Lib/site-packages/datasets/fingerprint.py b/venv/Lib/site-packages/datasets/fingerprint.py
new file mode 100644
index 0000000000000000000000000000000000000000..13b801621bb5945c61c34f615e9dfa5ab70234ae
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/fingerprint.py
@@ -0,0 +1,480 @@
+import inspect
+import os
+import random
+import shutil
+import tempfile
+import weakref
+from functools import wraps
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Callable, Optional, Union
+
+import numpy as np
+import xxhash
+
+from . import config
+from .naming import INVALID_WINDOWS_CHARACTERS_IN_PATH
+from .utils._dill import dumps
+from .utils.logging import get_logger
+
+
+if TYPE_CHECKING:
+ from .arrow_dataset import Dataset
+
+
+logger = get_logger(__name__)
+
+
+# Fingerprinting allows to have one deterministic fingerprint per dataset state.
+# A dataset fingerprint is updated after each transform.
+# Re-running the same transforms on a dataset in a different session results in the same fingerprint.
+# This is possible thanks to a custom hashing function that works with most python objects.
+
+# Fingerprinting is the main mechanism that enables caching.
+# The caching mechanism allows to reload an existing cache file if it's already been computed.
+
+
+#################
+# Caching
+#################
+
+_CACHING_ENABLED = True
+_TEMP_DIR_FOR_TEMP_CACHE_FILES: Optional["_TempCacheDir"] = None
+_DATASETS_WITH_TABLE_IN_TEMP_DIR: Optional[weakref.WeakSet] = None
+
+
+class _TempCacheDir:
+ """
+ A temporary directory for storing cached Arrow files with a cleanup that frees references to the Arrow files
+ before deleting the directory itself to avoid permission errors on Windows.
+ """
+
+ def __init__(self):
+ # Check if TMPDIR is set and handle the case where it doesn't exist
+ tmpdir = os.environ.get("TMPDIR") or os.environ.get("TEMP") or os.environ.get("TMP")
+ # Normalize the path to handle any path resolution issues
+ if tmpdir:
+ tmpdir = os.path.normpath(tmpdir)
+ if not os.path.exists(tmpdir):
+ # Auto-create the directory if it doesn't exist
+ # This prevents tempfile from silently falling back to /tmp
+ try:
+ os.makedirs(tmpdir, exist_ok=True)
+ logger.info(f"Created TMPDIR directory: {tmpdir}")
+ except OSError as e:
+ raise OSError(
+ f"TMPDIR is set to '{tmpdir}' but the directory does not exist and could not be created: {e}. "
+ "Please create it manually or unset TMPDIR to fall back to the default temporary directory."
+ ) from e
+ # If tmpdir exists, verify it's actually a directory and writable
+ elif not os.path.isdir(tmpdir):
+ raise OSError(
+ f"TMPDIR is set to '{tmpdir}' but it is not a directory. "
+ "Please point TMPDIR to a writable directory or unset it to fall back to the default temporary directory."
+ )
+
+ # Explicitly pass the directory to mkdtemp to ensure TMPDIR is respected
+ # This works even if tempfile.gettempdir() was already called and cached
+ # Pass dir=None if tmpdir is None to use default temp directory
+ self.name = tempfile.mkdtemp(prefix=config.TEMP_CACHE_DIR_PREFIX, dir=tmpdir)
+ self._finalizer = weakref.finalize(self, self._cleanup)
+
+ def _cleanup(self):
+ for dset in get_datasets_with_cache_file_in_temp_dir():
+ dset.__del__()
+ if os.path.exists(self.name):
+ try:
+ shutil.rmtree(self.name)
+ except Exception as e:
+ raise OSError(
+ f"An error occurred while trying to delete temporary cache directory {self.name}. Please delete it manually."
+ ) from e
+
+ def cleanup(self):
+ if self._finalizer.detach():
+ self._cleanup()
+
+
+def maybe_register_dataset_for_temp_dir_deletion(dataset):
+ """
+ This function registers the datasets that have cache files in _TEMP_DIR_FOR_TEMP_CACHE_FILES in order
+ to properly delete them before deleting the temporary directory.
+ The temporary directory _TEMP_DIR_FOR_TEMP_CACHE_FILES is used when caching is disabled.
+ """
+ if _TEMP_DIR_FOR_TEMP_CACHE_FILES is None:
+ return
+
+ global _DATASETS_WITH_TABLE_IN_TEMP_DIR
+ if _DATASETS_WITH_TABLE_IN_TEMP_DIR is None:
+ _DATASETS_WITH_TABLE_IN_TEMP_DIR = weakref.WeakSet()
+ if any(
+ Path(_TEMP_DIR_FOR_TEMP_CACHE_FILES.name) in Path(cache_file["filename"]).parents
+ for cache_file in dataset.cache_files
+ ):
+ _DATASETS_WITH_TABLE_IN_TEMP_DIR.add(dataset)
+
+
+def get_datasets_with_cache_file_in_temp_dir():
+ return list(_DATASETS_WITH_TABLE_IN_TEMP_DIR) if _DATASETS_WITH_TABLE_IN_TEMP_DIR is not None else []
+
+
+def enable_caching():
+ """
+ When applying transforms on a dataset, the data are stored in cache files.
+ The caching mechanism allows to reload an existing cache file if it's already been computed.
+
+ Reloading a dataset is possible since the cache files are named using the dataset fingerprint, which is updated
+ after each transform.
+
+ If disabled, the library will no longer reload cached datasets files when applying transforms to the datasets.
+ More precisely, if the caching is disabled:
+ - cache files are always recreated
+ - cache files are written to a temporary directory that is deleted when session closes
+ - cache files are named using a random hash instead of the dataset fingerprint
+ - use [`~datasets.Dataset.save_to_disk`] to save a transformed dataset or it will be deleted when session closes
+ - caching doesn't affect [`~datasets.load_dataset`]. If you want to regenerate a dataset from scratch you should use
+ the `download_mode` parameter in [`~datasets.load_dataset`].
+ """
+ global _CACHING_ENABLED
+ _CACHING_ENABLED = True
+
+
+def disable_caching():
+ """
+ When applying transforms on a dataset, the data are stored in cache files.
+ The caching mechanism allows to reload an existing cache file if it's already been computed.
+
+ Reloading a dataset is possible since the cache files are named using the dataset fingerprint, which is updated
+ after each transform.
+
+ If disabled, the library will no longer reload cached datasets files when applying transforms to the datasets.
+ More precisely, if the caching is disabled:
+ - cache files are always recreated
+ - cache files are written to a temporary directory that is deleted when session closes
+ - cache files are named using a random hash instead of the dataset fingerprint
+ - use [`~datasets.Dataset.save_to_disk`] to save a transformed dataset or it will be deleted when session closes
+ - caching doesn't affect [`~datasets.load_dataset`]. If you want to regenerate a dataset from scratch you should use
+ the `download_mode` parameter in [`~datasets.load_dataset`].
+ """
+ global _CACHING_ENABLED
+ _CACHING_ENABLED = False
+
+
+def is_caching_enabled() -> bool:
+ """
+ When applying transforms on a dataset, the data are stored in cache files.
+ The caching mechanism allows to reload an existing cache file if it's already been computed.
+
+ Reloading a dataset is possible since the cache files are named using the dataset fingerprint, which is updated
+ after each transform.
+
+ If disabled, the library will no longer reload cached datasets files when applying transforms to the datasets.
+ More precisely, if the caching is disabled:
+ - cache files are always recreated
+ - cache files are written to a temporary directory that is deleted when session closes
+ - cache files are named using a random hash instead of the dataset fingerprint
+ - use [`~datasets.Dataset.save_to_disk`]] to save a transformed dataset or it will be deleted when session closes
+ - caching doesn't affect [`~datasets.load_dataset`]. If you want to regenerate a dataset from scratch you should use
+ the `download_mode` parameter in [`~datasets.load_dataset`].
+ """
+ global _CACHING_ENABLED
+ return bool(_CACHING_ENABLED)
+
+
+def get_temporary_cache_files_directory() -> str:
+ """Return a directory that is deleted when session closes."""
+ global _TEMP_DIR_FOR_TEMP_CACHE_FILES
+ if _TEMP_DIR_FOR_TEMP_CACHE_FILES is None:
+ _TEMP_DIR_FOR_TEMP_CACHE_FILES = _TempCacheDir()
+ return _TEMP_DIR_FOR_TEMP_CACHE_FILES.name
+
+
+#################
+# Hashing
+#################
+
+
+class Hasher:
+ """Hasher that accepts python objects as inputs."""
+
+ dispatch: dict = {}
+
+ def __init__(self):
+ self.m = xxhash.xxh64()
+
+ @classmethod
+ def hash_bytes(cls, value: Union[bytes, list[bytes]]) -> str:
+ value = [value] if isinstance(value, bytes) else value
+ m = xxhash.xxh64()
+ for x in value:
+ m.update(x)
+ return m.hexdigest()
+
+ @classmethod
+ def hash(cls, value: Any) -> str:
+ return cls.hash_bytes(dumps(value))
+
+ def update(self, value: Any) -> None:
+ header_for_update = f"=={type(value)}=="
+ value_for_update = self.hash(value)
+ self.m.update(header_for_update.encode("utf8"))
+ self.m.update(value_for_update.encode("utf-8"))
+
+ def hexdigest(self) -> str:
+ return self.m.hexdigest()
+
+
+#################
+# Fingerprinting
+#################
+
+fingerprint_rng = random.Random()
+# we show a warning only once when fingerprinting fails to avoid spam
+fingerprint_warnings: dict[str, bool] = {}
+
+
+def generate_fingerprint(dataset: "Dataset") -> str:
+ state = dataset.__dict__
+ hasher = Hasher()
+ for key in sorted(state):
+ if key == "_fingerprint":
+ continue
+ hasher.update(key)
+ hasher.update(state[key])
+ # hash data files last modification timestamps as well
+ for cache_file in dataset.cache_files:
+ hasher.update(os.path.getmtime(cache_file["filename"]))
+ return hasher.hexdigest()
+
+
+def generate_random_fingerprint(nbits: int = 64) -> str:
+ return f"{fingerprint_rng.getrandbits(nbits):0{nbits // 4}x}"
+
+
+def update_fingerprint(fingerprint, transform, transform_args):
+ global fingerprint_warnings
+ hasher = Hasher()
+ hasher.update(fingerprint)
+ try:
+ hasher.update(transform)
+ except: # noqa various errors might raise here from pickle or dill
+ if _CACHING_ENABLED:
+ if not fingerprint_warnings.get("update_fingerprint_transform_hash_failed", False):
+ logger.warning(
+ f"Transform {transform} couldn't be hashed properly, a random hash was used instead. "
+ "Make sure your transforms and parameters are serializable with pickle or dill for the dataset fingerprinting and caching to work. "
+ "If you reuse this transform, the caching mechanism will consider it to be different from the previous calls and recompute everything. "
+ "This warning is only shown once. Subsequent hashing failures won't be shown."
+ )
+ fingerprint_warnings["update_fingerprint_transform_hash_failed"] = True
+ else:
+ logger.info(f"Transform {transform} couldn't be hashed properly, a random hash was used instead.")
+ else:
+ logger.info(
+ f"Transform {transform} couldn't be hashed properly, a random hash was used instead. This doesn't affect caching since it's disabled."
+ )
+
+ return generate_random_fingerprint()
+ for key in sorted(transform_args):
+ hasher.update(key)
+ try:
+ hasher.update(transform_args[key])
+ except: # noqa various errors might raise here from pickle or dill
+ if _CACHING_ENABLED:
+ if not fingerprint_warnings.get("update_fingerprint_transform_hash_failed", False):
+ logger.warning(
+ f"Parameter '{key}'={transform_args[key]} of the transform {transform} couldn't be hashed properly, a random hash was used instead. "
+ "Make sure your transforms and parameters are serializable with pickle or dill for the dataset fingerprinting and caching to work. "
+ "If you reuse this transform, the caching mechanism will consider it to be different from the previous calls and recompute everything. "
+ "This warning is only shown once. Subsequent hashing failures won't be shown."
+ )
+ fingerprint_warnings["update_fingerprint_transform_hash_failed"] = True
+ else:
+ logger.info(
+ f"Parameter '{key}'={transform_args[key]} of the transform {transform} couldn't be hashed properly, a random hash was used instead."
+ )
+ else:
+ logger.info(
+ f"Parameter '{key}'={transform_args[key]} of the transform {transform} couldn't be hashed properly, a random hash was used instead. This doesn't affect caching since it's disabled."
+ )
+ return generate_random_fingerprint()
+ return hasher.hexdigest()
+
+
+def validate_fingerprint(fingerprint: str, max_length=64):
+ """
+ Make sure the fingerprint is a non-empty string that is not longer that max_length=64 by default,
+ so that the fingerprint can be used to name cache files without issues.
+ """
+ if not isinstance(fingerprint, str) or not fingerprint:
+ raise ValueError(f"Invalid fingerprint '{fingerprint}': it should be a non-empty string.")
+ for invalid_char in INVALID_WINDOWS_CHARACTERS_IN_PATH:
+ if invalid_char in fingerprint:
+ raise ValueError(
+ f"Invalid fingerprint. Bad characters from black list '{INVALID_WINDOWS_CHARACTERS_IN_PATH}' found in '{fingerprint}'. "
+ f"They could create issues when creating cache files."
+ )
+ if len(fingerprint) > max_length:
+ raise ValueError(
+ f"Invalid fingerprint. Maximum lenth is {max_length} but '{fingerprint}' has length {len(fingerprint)}."
+ "It could create issues when creating cache files."
+ )
+
+
+def format_transform_for_fingerprint(func: Callable, version: Optional[str] = None) -> str:
+ """
+ Format a transform to the format that will be used to update the fingerprint.
+ """
+ transform = f"{func.__module__}.{func.__qualname__}"
+ if version is not None:
+ transform += f"@{version}"
+ return transform
+
+
+def format_kwargs_for_fingerprint(
+ func: Callable,
+ args: tuple,
+ kwargs: dict[str, Any],
+ use_kwargs: Optional[list[str]] = None,
+ ignore_kwargs: Optional[list[str]] = None,
+ randomized_function: bool = False,
+) -> dict[str, Any]:
+ """
+ Format the kwargs of a transform to the format that will be used to update the fingerprint.
+ """
+ kwargs_for_fingerprint = kwargs.copy()
+ if args:
+ params = [p.name for p in inspect.signature(func).parameters.values() if p != p.VAR_KEYWORD]
+ args = args[1:] # assume the first argument is the dataset
+ params = params[1:]
+ kwargs_for_fingerprint.update(zip(params, args))
+ else:
+ del kwargs_for_fingerprint[
+ next(iter(inspect.signature(func).parameters))
+ ] # assume the first key is the dataset
+
+ # keep the right kwargs to be hashed to generate the fingerprint
+
+ if use_kwargs:
+ kwargs_for_fingerprint = {k: v for k, v in kwargs_for_fingerprint.items() if k in use_kwargs}
+ if ignore_kwargs:
+ kwargs_for_fingerprint = {k: v for k, v in kwargs_for_fingerprint.items() if k not in ignore_kwargs}
+ if randomized_function: # randomized functions have `seed` and `generator` parameters
+ if kwargs_for_fingerprint.get("seed") is None and kwargs_for_fingerprint.get("generator") is None:
+ _, seed, pos, *_ = np.random.get_state()
+ seed = seed[pos] if pos < 624 else seed[0]
+ kwargs_for_fingerprint["generator"] = np.random.default_rng(seed)
+
+ # remove kwargs that are the default values
+
+ default_values = {
+ p.name: p.default for p in inspect.signature(func).parameters.values() if p.default != inspect._empty
+ }
+ for default_varname, default_value in default_values.items():
+ if default_varname in kwargs_for_fingerprint and kwargs_for_fingerprint[default_varname] == default_value:
+ kwargs_for_fingerprint.pop(default_varname)
+ return kwargs_for_fingerprint
+
+
+def fingerprint_transform(
+ inplace: bool,
+ use_kwargs: Optional[list[str]] = None,
+ ignore_kwargs: Optional[list[str]] = None,
+ fingerprint_names: Optional[list[str]] = None,
+ randomized_function: bool = False,
+ version: Optional[str] = None,
+):
+ """
+ Wrapper for dataset transforms to update the dataset fingerprint using ``update_fingerprint``
+ Args:
+ inplace (:obj:`bool`): If inplace is True, the fingerprint of the dataset is updated inplace.
+ Otherwise, a parameter "new_fingerprint" is passed to the wrapped method that should take care of
+ setting the fingerprint of the returned Dataset.
+ use_kwargs (:obj:`List[str]`, optional): optional white list of argument names to take into account
+ to update the fingerprint to the wrapped method that should take care of
+ setting the fingerprint of the returned Dataset. By default all the arguments are used.
+ ignore_kwargs (:obj:`List[str]`, optional): optional black list of argument names to take into account
+ to update the fingerprint. Note that ignore_kwargs prevails on use_kwargs.
+ fingerprint_names (:obj:`List[str]`, optional, defaults to ["new_fingerprint"]):
+ If the dataset transforms is not inplace and returns a DatasetDict, then it can require
+ several fingerprints (one per dataset in the DatasetDict). By specifying fingerprint_names,
+ one fingerprint named after each element of fingerprint_names is going to be passed.
+ randomized_function (:obj:`bool`, defaults to False): If the dataset transform is random and has
+ optional parameters "seed" and "generator", then you can set randomized_function to True.
+ This way, even if users set "seed" and "generator" to None, then the fingerprint is
+ going to be randomly generated depending on numpy's current state. In this case, the
+ generator is set to np.random.default_rng(np.random.get_state()[1][0]).
+ version (:obj:`str`, optional): version of the transform. The version is taken into account when
+ computing the fingerprint. If a datase transform changes (or at least if the output data
+ that are cached changes), then one should increase the version. If the version stays the
+ same, then old cached data could be reused that are not compatible with the new transform.
+ It should be in the format "MAJOR.MINOR.PATCH".
+ """
+
+ if use_kwargs is not None and not isinstance(use_kwargs, list):
+ raise ValueError(f"use_kwargs is supposed to be a list, not {type(use_kwargs)}")
+
+ if ignore_kwargs is not None and not isinstance(ignore_kwargs, list):
+ raise ValueError(f"ignore_kwargs is supposed to be a list, not {type(use_kwargs)}")
+
+ if inplace and fingerprint_names:
+ raise ValueError("fingerprint_names are only used when inplace is False")
+
+ fingerprint_names = fingerprint_names if fingerprint_names is not None else ["new_fingerprint"]
+
+ def _fingerprint(func):
+ if not inplace and not all(name in func.__code__.co_varnames for name in fingerprint_names):
+ raise ValueError(f"function {func} is missing parameters {fingerprint_names} in signature")
+
+ if randomized_function: # randomized function have seed and generator parameters
+ if "seed" not in func.__code__.co_varnames:
+ raise ValueError(f"'seed' must be in {func}'s signature")
+ if "generator" not in func.__code__.co_varnames:
+ raise ValueError(f"'generator' must be in {func}'s signature")
+ # this call has to be outside the wrapper or since __qualname__ changes in multiprocessing
+ transform = format_transform_for_fingerprint(func, version=version)
+
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ kwargs_for_fingerprint = format_kwargs_for_fingerprint(
+ func,
+ args,
+ kwargs,
+ use_kwargs=use_kwargs,
+ ignore_kwargs=ignore_kwargs,
+ randomized_function=randomized_function,
+ )
+
+ if args:
+ dataset: Dataset = args[0]
+ args = args[1:]
+ else:
+ dataset: Dataset = kwargs.pop(next(iter(inspect.signature(func).parameters)))
+
+ # compute new_fingerprint and add it to the args of not in-place transforms
+ if inplace:
+ new_fingerprint = update_fingerprint(dataset._fingerprint, transform, kwargs_for_fingerprint)
+ else:
+ for fingerprint_name in fingerprint_names: # transforms like `train_test_split` have several hashes
+ if kwargs.get(fingerprint_name) is None:
+ kwargs_for_fingerprint["fingerprint_name"] = fingerprint_name
+ kwargs[fingerprint_name] = update_fingerprint(
+ dataset._fingerprint, transform, kwargs_for_fingerprint
+ )
+ else:
+ validate_fingerprint(kwargs[fingerprint_name])
+
+ # Call actual function
+
+ out = func(dataset, *args, **kwargs)
+
+ # Update fingerprint of in-place transforms + update in-place history of transforms
+
+ if inplace: # update after calling func so that the fingerprint doesn't change if the function fails
+ dataset._fingerprint = new_fingerprint
+
+ return out
+
+ wrapper._decorator_name_ = "fingerprint"
+ return wrapper
+
+ return _fingerprint
diff --git a/venv/Lib/site-packages/datasets/formatting/__init__.py b/venv/Lib/site-packages/datasets/formatting/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e29421df342f172cece9e28ff619c7760b34c6a0
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/formatting/__init__.py
@@ -0,0 +1,136 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict, List, Optional, Type
+
+from .. import config
+from ..utils import logging
+from .formatting import (
+ ArrowFormatter,
+ CustomFormatter,
+ Formatter,
+ PandasFormatter,
+ PythonFormatter,
+ TableFormatter,
+ TensorFormatter,
+ format_table,
+ query_table,
+)
+from .np_formatter import NumpyFormatter
+
+
+logger = logging.get_logger(__name__)
+
+_FORMAT_TYPES: dict[Optional[str], type[Formatter]] = {}
+_FORMAT_TYPES_ALIASES: dict[Optional[str], str] = {}
+_FORMAT_TYPES_ALIASES_UNAVAILABLE: dict[Optional[str], Exception] = {}
+
+
+def _register_formatter(
+ formatter_cls: type,
+ format_type: Optional[str],
+ aliases: Optional[list[str]] = None,
+):
+ """
+ Register a Formatter object using a name and optional aliases.
+ This function must be used on a Formatter class.
+ """
+ aliases = aliases if aliases is not None else []
+ if format_type in _FORMAT_TYPES:
+ logger.warning(
+ f"Overwriting format type '{format_type}' ({_FORMAT_TYPES[format_type].__name__} -> {formatter_cls.__name__})"
+ )
+ _FORMAT_TYPES[format_type] = formatter_cls
+ for alias in set(aliases + [format_type]):
+ if alias in _FORMAT_TYPES_ALIASES:
+ logger.warning(
+ f"Overwriting format type alias '{alias}' ({_FORMAT_TYPES_ALIASES[alias]} -> {format_type})"
+ )
+ _FORMAT_TYPES_ALIASES[alias] = format_type
+
+
+def _register_unavailable_formatter(
+ unavailable_error: Exception, format_type: Optional[str], aliases: Optional[list[str]] = None
+):
+ """
+ Register an unavailable Formatter object using a name and optional aliases.
+ This function must be used on an Exception object that is raised when trying to get the unavailable formatter.
+ """
+ aliases = aliases if aliases is not None else []
+ for alias in set(aliases + [format_type]):
+ _FORMAT_TYPES_ALIASES_UNAVAILABLE[alias] = unavailable_error
+
+
+# Here we define all the available formatting functions that can be used by `Dataset.set_format`
+_register_formatter(PythonFormatter, None, aliases=["python"])
+_register_formatter(ArrowFormatter, "arrow", aliases=["pa", "pyarrow"])
+_register_formatter(NumpyFormatter, "numpy", aliases=["np"])
+_register_formatter(PandasFormatter, "pandas", aliases=["pd"])
+_register_formatter(CustomFormatter, "custom")
+
+if config.POLARS_AVAILABLE:
+ from .polars_formatter import PolarsFormatter
+
+ _register_formatter(PolarsFormatter, "polars", aliases=["pl"])
+else:
+ _polars_error = ValueError("Polars needs to be installed to be able to return Polars dataframes.")
+ _register_unavailable_formatter(_polars_error, "polars", aliases=["pl"])
+
+if config.TORCH_AVAILABLE:
+ from .torch_formatter import TorchFormatter
+
+ _register_formatter(TorchFormatter, "torch", aliases=["pt", "pytorch"])
+else:
+ _torch_error = ValueError("PyTorch needs to be installed to be able to return PyTorch tensors.")
+ _register_unavailable_formatter(_torch_error, "torch", aliases=["pt", "pytorch"])
+
+if config.TF_AVAILABLE:
+ from .tf_formatter import TFFormatter
+
+ _register_formatter(TFFormatter, "tensorflow", aliases=["tf"])
+else:
+ _tf_error = ValueError("Tensorflow needs to be installed to be able to return Tensorflow tensors.")
+ _register_unavailable_formatter(_tf_error, "tensorflow", aliases=["tf"])
+
+if config.JAX_AVAILABLE:
+ from .jax_formatter import JaxFormatter
+
+ _register_formatter(JaxFormatter, "jax", aliases=[])
+else:
+ _jax_error = ValueError("JAX needs to be installed to be able to return JAX arrays.")
+ _register_unavailable_formatter(_jax_error, "jax", aliases=[])
+
+
+def get_format_type_from_alias(format_type: Optional[str]) -> Optional[str]:
+ """If the given format type is a known alias, then return its main type name. Otherwise return the type with no change."""
+ if format_type in _FORMAT_TYPES_ALIASES:
+ return _FORMAT_TYPES_ALIASES[format_type]
+ else:
+ return format_type
+
+
+def get_formatter(format_type: Optional[str], **format_kwargs) -> Formatter:
+ """
+ Factory function to get a Formatter given its type name and keyword arguments.
+ A formatter is an object that extracts and formats data from pyarrow table.
+ It defines the formatting for rows, columns and batches.
+ If the formatter for a given type name doesn't exist or is not available, an error is raised.
+ """
+ format_type = get_format_type_from_alias(format_type)
+ if format_type in _FORMAT_TYPES:
+ return _FORMAT_TYPES[format_type](**format_kwargs)
+ if format_type in _FORMAT_TYPES_ALIASES_UNAVAILABLE:
+ raise _FORMAT_TYPES_ALIASES_UNAVAILABLE[format_type]
+ else:
+ raise ValueError(f"Format type should be one of {list(_FORMAT_TYPES.keys())}, but got '{format_type}'")
diff --git a/venv/Lib/site-packages/datasets/formatting/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/formatting/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ef1c5e40ffff7290b268150f85db768ffb2cf59d
Binary files /dev/null and b/venv/Lib/site-packages/datasets/formatting/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/formatting/__pycache__/formatting.cpython-311.pyc b/venv/Lib/site-packages/datasets/formatting/__pycache__/formatting.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..09b1e65f4e1b3fa46b0f76ddeaa4d410f761de6e
Binary files /dev/null and b/venv/Lib/site-packages/datasets/formatting/__pycache__/formatting.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/formatting/__pycache__/jax_formatter.cpython-311.pyc b/venv/Lib/site-packages/datasets/formatting/__pycache__/jax_formatter.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c52e657cd53d3445561d0dd4ac5b2b3e376d1202
Binary files /dev/null and b/venv/Lib/site-packages/datasets/formatting/__pycache__/jax_formatter.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/formatting/__pycache__/np_formatter.cpython-311.pyc b/venv/Lib/site-packages/datasets/formatting/__pycache__/np_formatter.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b4797ff9df6d61480877479c748adf55fd1b5f64
Binary files /dev/null and b/venv/Lib/site-packages/datasets/formatting/__pycache__/np_formatter.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/formatting/__pycache__/polars_formatter.cpython-311.pyc b/venv/Lib/site-packages/datasets/formatting/__pycache__/polars_formatter.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0c22990125905fce4a350a8a5ecdd8a90a99ed68
Binary files /dev/null and b/venv/Lib/site-packages/datasets/formatting/__pycache__/polars_formatter.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/formatting/__pycache__/tf_formatter.cpython-311.pyc b/venv/Lib/site-packages/datasets/formatting/__pycache__/tf_formatter.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e21825e6ce1c1cf71fdd6b78137beb0575a5bba1
Binary files /dev/null and b/venv/Lib/site-packages/datasets/formatting/__pycache__/tf_formatter.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/formatting/__pycache__/torch_formatter.cpython-311.pyc b/venv/Lib/site-packages/datasets/formatting/__pycache__/torch_formatter.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f47a0131502c58f07dfee0388b6c54cdf6fc20b1
Binary files /dev/null and b/venv/Lib/site-packages/datasets/formatting/__pycache__/torch_formatter.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/formatting/formatting.py b/venv/Lib/site-packages/datasets/formatting/formatting.py
new file mode 100644
index 0000000000000000000000000000000000000000..24805dc1e79a22c6639091d45f9d4ba46ecfa193
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/formatting/formatting.py
@@ -0,0 +1,678 @@
+# Copyright 2020 The HuggingFace Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numbers
+import operator
+from collections.abc import Iterable, Mapping, MutableMapping
+from functools import partial
+
+# Lint as: python3
+from typing import Any, Callable, Generic, Optional, TypeVar, Union
+
+import numpy as np
+import pandas as pd
+import pyarrow as pa
+
+from ..features import Features
+from ..features.features import _ArrayXDExtensionType, _is_zero_copy_only, decode_nested_example, pandas_types_mapper
+from ..table import Table
+from ..utils.py_utils import no_op_if_value_is_null
+
+
+T = TypeVar("T")
+
+RowFormat = TypeVar("RowFormat")
+ColumnFormat = TypeVar("ColumnFormat")
+BatchFormat = TypeVar("BatchFormat")
+
+
+def _is_range_contiguous(key: range) -> bool:
+ return key.step == 1 and key.stop >= key.start
+
+
+def _raise_bad_key_type(key: Any):
+ raise TypeError(
+ f"Wrong key type: '{key}' of type '{type(key)}'. Expected one of int, slice, range, str or Iterable."
+ )
+
+
+def _query_table_with_indices_mapping(
+ table: Table, key: Union[int, slice, range, str, Iterable], indices: Table
+) -> pa.Table:
+ """
+ Query a pyarrow Table to extract the subtable that correspond to the given key.
+ The :obj:`indices` parameter corresponds to the indices mapping in case we cant to take into
+ account a shuffling or an indices selection for example.
+ The indices table must contain one column named "indices" of type uint64.
+ """
+ if isinstance(key, int):
+ key = indices.fast_slice(key % indices.num_rows, 1).column(0)[0].as_py()
+ return _query_table(table, key)
+ if isinstance(key, slice):
+ key = range(*key.indices(indices.num_rows))
+ if isinstance(key, range):
+ if _is_range_contiguous(key) and key.start >= 0:
+ return _query_table(
+ table, [i.as_py() for i in indices.fast_slice(key.start, key.stop - key.start).column(0)]
+ )
+ else:
+ pass # treat as an iterable
+ if isinstance(key, str):
+ table = table.select([key])
+ return _query_table(table, indices.column(0).to_pylist())
+ if isinstance(key, Iterable):
+ return _query_table(table, [indices.fast_slice(i, 1).column(0)[0].as_py() for i in key])
+
+ _raise_bad_key_type(key)
+
+
+def _query_table(table: Table, key: Union[int, slice, range, str, Iterable]) -> pa.Table:
+ """
+ Query a pyarrow Table to extract the subtable that correspond to the given key.
+ """
+ if isinstance(key, int):
+ return table.fast_slice(key % table.num_rows, 1)
+ if isinstance(key, slice):
+ key = range(*key.indices(table.num_rows))
+ if isinstance(key, range):
+ if _is_range_contiguous(key) and key.start >= 0:
+ return table.fast_slice(key.start, key.stop - key.start)
+ else:
+ pass # treat as an iterable
+ if isinstance(key, str):
+ return table.table.drop([column for column in table.column_names if column != key])
+ if isinstance(key, Iterable):
+ key = np.fromiter(key, np.int64)
+ if len(key) == 0:
+ return table.table.slice(0, 0)
+ # don't use pyarrow.Table.take even for pyarrow >=1.0 (see https://issues.apache.org/jira/browse/ARROW-9773)
+ return table.fast_gather(key % table.num_rows)
+
+ _raise_bad_key_type(key)
+
+
+def _is_array_with_nulls(pa_array: pa.Array) -> bool:
+ return pa_array.null_count > 0
+
+
+class BaseArrowExtractor(Generic[RowFormat, ColumnFormat, BatchFormat]):
+ """
+ Arrow extractor are used to extract data from pyarrow tables.
+ It makes it possible to extract rows, columns and batches.
+ These three extractions types have to be implemented.
+ """
+
+ def extract_row(self, pa_table: pa.Table) -> RowFormat:
+ raise NotImplementedError
+
+ def extract_column(self, pa_table: pa.Table) -> ColumnFormat:
+ raise NotImplementedError
+
+ def extract_batch(self, pa_table: pa.Table) -> BatchFormat:
+ raise NotImplementedError
+
+
+def _unnest(py_dict: dict[str, list[T]]) -> dict[str, T]:
+ """Return the first element of a batch (dict) as a row (dict)"""
+ return {key: array[0] for key, array in py_dict.items()}
+
+
+class SimpleArrowExtractor(BaseArrowExtractor[pa.Table, pa.Array, pa.Table]):
+ def extract_row(self, pa_table: pa.Table) -> pa.Table:
+ return pa_table
+
+ def extract_column(self, pa_table: pa.Table) -> pa.Array:
+ return pa_table.column(0)
+
+ def extract_batch(self, pa_table: pa.Table) -> pa.Table:
+ return pa_table
+
+
+class PythonArrowExtractor(BaseArrowExtractor[dict, list, dict]):
+ def extract_row(self, pa_table: pa.Table) -> dict:
+ return _unnest(pa_table.to_pydict())
+
+ def extract_column(self, pa_table: pa.Table) -> list:
+ return pa_table.column(0).to_pylist()
+
+ def extract_batch(self, pa_table: pa.Table) -> dict:
+ return pa_table.to_pydict()
+
+
+class NumpyArrowExtractor(BaseArrowExtractor[dict, np.ndarray, dict]):
+ def __init__(self, **np_array_kwargs):
+ self.np_array_kwargs = np_array_kwargs
+
+ def extract_row(self, pa_table: pa.Table) -> dict:
+ return _unnest(self.extract_batch(pa_table))
+
+ def extract_column(self, pa_table: pa.Table) -> np.ndarray:
+ return self._arrow_array_to_numpy(pa_table[pa_table.column_names[0]])
+
+ def extract_batch(self, pa_table: pa.Table) -> dict:
+ return {col: self._arrow_array_to_numpy(pa_table[col]) for col in pa_table.column_names}
+
+ def _arrow_array_to_numpy(self, pa_array: pa.Array) -> np.ndarray:
+ if isinstance(pa_array, pa.ChunkedArray):
+ if isinstance(pa_array.type, _ArrayXDExtensionType):
+ # don't call to_pylist() to preserve dtype of the fixed-size array
+ zero_copy_only = _is_zero_copy_only(pa_array.type.storage_dtype, unnest=True)
+ array: list = [
+ row for chunk in pa_array.chunks for row in chunk.to_numpy(zero_copy_only=zero_copy_only)
+ ]
+ else:
+ zero_copy_only = _is_zero_copy_only(pa_array.type) and all(
+ not _is_array_with_nulls(chunk) for chunk in pa_array.chunks
+ )
+ array: list = [
+ row for chunk in pa_array.chunks for row in chunk.to_numpy(zero_copy_only=zero_copy_only)
+ ]
+ else:
+ if isinstance(pa_array.type, _ArrayXDExtensionType):
+ # don't call to_pylist() to preserve dtype of the fixed-size array
+ zero_copy_only = _is_zero_copy_only(pa_array.type.storage_dtype, unnest=True)
+ array: list = pa_array.to_numpy(zero_copy_only=zero_copy_only)
+ else:
+ zero_copy_only = _is_zero_copy_only(pa_array.type) and not _is_array_with_nulls(pa_array)
+ array: list = pa_array.to_numpy(zero_copy_only=zero_copy_only).tolist()
+
+ if len(array) > 0:
+ if any(
+ (isinstance(x, np.ndarray) and (x.dtype == object or x.shape != array[0].shape))
+ or (isinstance(x, float) and np.isnan(x))
+ for x in array
+ ):
+ if np.lib.NumpyVersion(np.__version__) >= "2.0.0b1":
+ return np.asarray(array, dtype=object)
+ return np.array(array, copy=False, dtype=object)
+ if np.lib.NumpyVersion(np.__version__) >= "2.0.0b1":
+ return np.asarray(array)
+ else:
+ return np.array(array, copy=False)
+
+
+class PandasArrowExtractor(BaseArrowExtractor[pd.DataFrame, pd.Series, pd.DataFrame]):
+ def extract_row(self, pa_table: pa.Table) -> pd.DataFrame:
+ return pa_table.slice(length=1).to_pandas(types_mapper=pandas_types_mapper)
+
+ def extract_column(self, pa_table: pa.Table) -> pd.Series:
+ return pa_table.select([0]).to_pandas(types_mapper=pandas_types_mapper)[pa_table.column_names[0]]
+
+ def extract_batch(self, pa_table: pa.Table) -> pd.DataFrame:
+ return pa_table.to_pandas(types_mapper=pandas_types_mapper)
+
+
+class PythonFeaturesDecoder:
+ def __init__(
+ self, features: Optional[Features], token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None
+ ):
+ self.features = features
+ self.token_per_repo_id = token_per_repo_id
+
+ def decode_row(self, row: dict) -> dict:
+ return self.features.decode_example(row, token_per_repo_id=self.token_per_repo_id) if self.features else row
+
+ def decode_column(self, column: list, column_name: str) -> list:
+ return (
+ self.features.decode_column(column, column_name, token_per_repo_id=self.token_per_repo_id)
+ if self.features
+ else column
+ )
+
+ def decode_batch(self, batch: dict) -> dict:
+ return self.features.decode_batch(batch, token_per_repo_id=self.token_per_repo_id) if self.features else batch
+
+
+class PandasFeaturesDecoder:
+ def __init__(self, features: Optional[Features]):
+ self.features = features
+
+ def decode_row(self, row: pd.DataFrame) -> pd.DataFrame:
+ decode = (
+ {
+ column_name: no_op_if_value_is_null(partial(decode_nested_example, feature))
+ for column_name, feature in self.features.items()
+ if self.features._column_requires_decoding[column_name]
+ }
+ if self.features
+ else {}
+ )
+ if decode:
+ row[list(decode.keys())] = row.transform(decode)
+ return row
+
+ def decode_column(self, column: pd.Series, column_name: str) -> pd.Series:
+ decode = (
+ no_op_if_value_is_null(partial(decode_nested_example, self.features[column_name]))
+ if self.features and column_name in self.features and self.features._column_requires_decoding[column_name]
+ else None
+ )
+ if decode:
+ column = column.transform(decode)
+ return column
+
+ def decode_batch(self, batch: pd.DataFrame) -> pd.DataFrame:
+ return self.decode_row(batch)
+
+
+class LazyDict(MutableMapping):
+ """A dictionary backed by Arrow data. The values are formatted on-the-fly when accessing the dictionary."""
+
+ def __init__(self, pa_table: pa.Table, formatter: "Formatter"):
+ self.pa_table = pa_table
+ self.formatter = formatter
+
+ self.data = dict.fromkeys(pa_table.column_names)
+ self.keys_to_format = set(self.data.keys())
+
+ def __len__(self):
+ return len(self.data)
+
+ def __getitem__(self, key):
+ value = self.data[key]
+ if key in self.keys_to_format:
+ value = self.format(key)
+ self.data[key] = value
+ self.keys_to_format.remove(key)
+ return value
+
+ def __setitem__(self, key, value):
+ if key in self.keys_to_format:
+ self.keys_to_format.remove(key)
+ self.data[key] = value
+
+ def __delitem__(self, key) -> None:
+ if key in self.keys_to_format:
+ self.keys_to_format.remove(key)
+ del self.data[key]
+
+ def __iter__(self):
+ return iter(self.data)
+
+ def __contains__(self, key):
+ return key in self.data
+
+ def __repr__(self):
+ self._format_all()
+ return repr(self.data)
+
+ def __or__(self, other):
+ if isinstance(other, LazyDict):
+ inst = self.copy()
+ other = other.copy()
+ other._format_all()
+ inst.keys_to_format -= other.data.keys()
+ inst.data = inst.data | other.data
+ return inst
+ if isinstance(other, dict):
+ inst = self.copy()
+ inst.keys_to_format -= other.keys()
+ inst.data = inst.data | other
+ return inst
+ return NotImplemented
+
+ def __ror__(self, other):
+ if isinstance(other, LazyDict):
+ inst = self.copy()
+ other = other.copy()
+ other._format_all()
+ inst.keys_to_format -= other.data.keys()
+ inst.data = other.data | inst.data
+ return inst
+ if isinstance(other, dict):
+ inst = self.copy()
+ inst.keys_to_format -= other.keys()
+ inst.data = other | inst.data
+ return inst
+ return NotImplemented
+
+ def __ior__(self, other):
+ if isinstance(other, LazyDict):
+ other = other.copy()
+ other._format_all()
+ self.keys_to_format -= other.data.keys()
+ self.data |= other.data
+ else:
+ self.keys_to_format -= other.keys()
+ self.data |= other
+ return self
+
+ def __copy__(self):
+ # Identical to `UserDict.__copy__`
+ inst = self.__class__.__new__(self.__class__)
+ inst.__dict__.update(self.__dict__)
+ # Create a copy and avoid triggering descriptors
+ inst.__dict__["data"] = self.__dict__["data"].copy()
+ inst.__dict__["keys_to_format"] = self.__dict__["keys_to_format"].copy()
+ return inst
+
+ def copy(self):
+ import copy
+
+ return copy.copy(self)
+
+ @classmethod
+ def fromkeys(cls, iterable, value=None):
+ raise NotImplementedError
+
+ def format(self, key):
+ raise NotImplementedError
+
+ def _format_all(self):
+ for key in self.keys_to_format:
+ self.data[key] = self.format(key)
+ self.keys_to_format.clear()
+
+
+class LazyRow(LazyDict):
+ def format(self, key):
+ return self.formatter.format_column(self.pa_table.select([key]))[0]
+
+
+class LazyBatch(LazyDict):
+ def format(self, key):
+ return self.formatter.format_column(self.pa_table.select([key]))
+
+
+class Formatter(Generic[RowFormat, ColumnFormat, BatchFormat]):
+ """
+ A formatter is an object that extracts and formats data from pyarrow tables.
+ It defines the formatting for rows, columns and batches.
+ """
+
+ simple_arrow_extractor = SimpleArrowExtractor
+ python_arrow_extractor = PythonArrowExtractor
+ numpy_arrow_extractor = NumpyArrowExtractor
+ pandas_arrow_extractor = PandasArrowExtractor
+
+ def __init__(
+ self,
+ features: Optional[Features] = None,
+ token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None,
+ ):
+ self.features = features
+ self.token_per_repo_id = token_per_repo_id
+ self.python_features_decoder = PythonFeaturesDecoder(self.features, self.token_per_repo_id)
+ self.pandas_features_decoder = PandasFeaturesDecoder(self.features)
+
+ def __call__(self, pa_table: pa.Table, query_type: str) -> Union[RowFormat, ColumnFormat, BatchFormat]:
+ if query_type == "row":
+ return self.format_row(pa_table)
+ elif query_type == "column":
+ return self.format_column(pa_table)
+ elif query_type == "batch":
+ return self.format_batch(pa_table)
+
+ def format_row(self, pa_table: pa.Table) -> RowFormat:
+ raise NotImplementedError
+
+ def format_column(self, pa_table: pa.Table) -> ColumnFormat:
+ raise NotImplementedError
+
+ def format_batch(self, pa_table: pa.Table) -> BatchFormat:
+ raise NotImplementedError
+
+
+class TensorFormatter(Formatter[RowFormat, ColumnFormat, BatchFormat]):
+ def recursive_tensorize(self, data_struct: dict):
+ raise NotImplementedError
+
+
+class TableFormatter(Formatter[RowFormat, ColumnFormat, BatchFormat]):
+ table_type: str
+ column_type: str
+
+
+class ArrowFormatter(TableFormatter[pa.Table, pa.Array, pa.Table]):
+ table_type = "arrow table"
+ column_type = "arrow array"
+
+ def format_row(self, pa_table: pa.Table) -> pa.Table:
+ return self.simple_arrow_extractor().extract_row(pa_table)
+
+ def format_column(self, pa_table: pa.Table) -> pa.Array:
+ return self.simple_arrow_extractor().extract_column(pa_table)
+
+ def format_batch(self, pa_table: pa.Table) -> pa.Table:
+ return self.simple_arrow_extractor().extract_batch(pa_table)
+
+
+class PythonFormatter(Formatter[Mapping, list, Mapping]):
+ def __init__(self, features=None, lazy=False, token_per_repo_id=None):
+ super().__init__(features, token_per_repo_id)
+ self.lazy = lazy
+
+ def format_row(self, pa_table: pa.Table) -> Mapping:
+ if self.lazy:
+ return LazyRow(pa_table, self)
+ row = self.python_arrow_extractor().extract_row(pa_table)
+ row = self.python_features_decoder.decode_row(row)
+ return row
+
+ def format_column(self, pa_table: pa.Table) -> list:
+ column = self.python_arrow_extractor().extract_column(pa_table)
+ column = self.python_features_decoder.decode_column(column, pa_table.column_names[0])
+ return column
+
+ def format_batch(self, pa_table: pa.Table) -> Mapping:
+ if self.lazy:
+ return LazyBatch(pa_table, self)
+ batch = self.python_arrow_extractor().extract_batch(pa_table)
+ batch = self.python_features_decoder.decode_batch(batch)
+ return batch
+
+
+class PandasFormatter(TableFormatter[pd.DataFrame, pd.Series, pd.DataFrame]):
+ table_type = "pandas dataframe"
+ column_type = "pandas series"
+
+ def format_row(self, pa_table: pa.Table) -> pd.DataFrame:
+ row = self.pandas_arrow_extractor().extract_row(pa_table)
+ row = self.pandas_features_decoder.decode_row(row)
+ return row
+
+ def format_column(self, pa_table: pa.Table) -> pd.Series:
+ column = self.pandas_arrow_extractor().extract_column(pa_table)
+ column = self.pandas_features_decoder.decode_column(column, pa_table.column_names[0])
+ return column
+
+ def format_batch(self, pa_table: pa.Table) -> pd.DataFrame:
+ row = self.pandas_arrow_extractor().extract_batch(pa_table)
+ row = self.pandas_features_decoder.decode_batch(row)
+ return row
+
+
+class CustomFormatter(Formatter[dict, ColumnFormat, dict]):
+ """
+ A user-defined custom formatter function defined by a ``transform``.
+ The transform must take as input a batch of data extracted for an arrow table using the python extractor,
+ and return a batch.
+ If the output batch is not a dict, then output_all_columns won't work.
+ If the output batch has several fields, then querying a single column won't work since we don't know which field
+ to return.
+ """
+
+ def __init__(self, transform: Callable[[dict], dict], features=None, token_per_repo_id=None, **kwargs):
+ super().__init__(features=features, token_per_repo_id=token_per_repo_id)
+ self.transform = transform
+
+ def format_row(self, pa_table: pa.Table) -> dict:
+ formatted_batch = self.format_batch(pa_table)
+ try:
+ return _unnest(formatted_batch)
+ except Exception as exc:
+ raise TypeError(
+ f"Custom formatting function must return a dict of sequences to be able to pick a row, but got {formatted_batch}"
+ ) from exc
+
+ def format_column(self, pa_table: pa.Table) -> ColumnFormat:
+ formatted_batch = self.format_batch(pa_table)
+ if hasattr(formatted_batch, "keys"):
+ if len(formatted_batch.keys()) > 1:
+ raise TypeError(
+ "Tried to query a column but the custom formatting function returns too many columns. "
+ f"Only one column was expected but got columns {list(formatted_batch.keys())}."
+ )
+ else:
+ raise TypeError(
+ f"Custom formatting function must return a dict to be able to pick a row, but got {formatted_batch}"
+ )
+ try:
+ return formatted_batch[pa_table.column_names[0]]
+ except Exception as exc:
+ raise TypeError(
+ f"Custom formatting function must return a dict to be able to pick a row, but got {formatted_batch}"
+ ) from exc
+
+ def format_batch(self, pa_table: pa.Table) -> dict:
+ batch = self.python_arrow_extractor().extract_batch(pa_table)
+ batch = self.python_features_decoder.decode_batch(batch)
+ return self.transform(batch)
+
+
+def _check_valid_column_key(key: str, columns: list[str]) -> None:
+ if key not in columns:
+ raise KeyError(f"Column {key} not in the dataset. Current columns in the dataset: {columns}")
+
+
+def _check_valid_index_key(key: Union[int, slice, range, Iterable], size: int) -> None:
+ if isinstance(key, int):
+ if (key < 0 and key + size < 0) or (key >= size):
+ raise IndexError(f"Invalid key: {key} is out of bounds for size {size}")
+ return
+ elif isinstance(key, slice):
+ pass
+ elif isinstance(key, range):
+ if len(key) > 0:
+ _check_valid_index_key(max(key), size=size)
+ _check_valid_index_key(min(key), size=size)
+ elif isinstance(key, Iterable):
+ if len(key) > 0:
+ _check_valid_index_key(int(max(key)), size=size)
+ _check_valid_index_key(int(min(key)), size=size)
+ else:
+ _raise_bad_key_type(key)
+
+
+def key_to_query_type(key: Union[int, slice, range, str, Iterable]) -> str:
+ if isinstance(key, numbers.Integral):
+ return "row"
+ elif isinstance(key, str):
+ return "column"
+ elif isinstance(key, (slice, range, Iterable)):
+ return "batch"
+ _raise_bad_key_type(key)
+
+
+def query_table(
+ table: Table,
+ key: Union[int, slice, range, str, Iterable],
+ indices: Optional[Table] = None,
+) -> pa.Table:
+ """
+ Query a Table to extract the subtable that correspond to the given key.
+
+ Args:
+ table (``datasets.table.Table``): The input Table to query from
+ key (``Union[int, slice, range, str, Iterable]``): The key can be of different types:
+ - an integer i: the subtable containing only the i-th row
+ - a slice [i:j:k]: the subtable containing the rows that correspond to this slice
+ - a range(i, j, k): the subtable containing the rows that correspond to this range
+ - a string c: the subtable containing all the rows but only the column c
+ - an iterable l: the subtable that is the concatenation of all the i-th rows for all i in the iterable
+ indices (Optional ``datasets.table.Table``): If not None, it is used to re-map the given key to the table rows.
+ The indices table must contain one column named "indices" of type uint64.
+ This is used in case of shuffling or rows selection.
+
+
+ Returns:
+ ``pyarrow.Table``: the result of the query on the input table
+ """
+ # Check if key is valid
+ if not isinstance(key, (int, slice, range, str, Iterable)):
+ try:
+ key = operator.index(key)
+ except TypeError:
+ _raise_bad_key_type(key)
+ if isinstance(key, str):
+ _check_valid_column_key(key, table.column_names)
+ else:
+ size = indices.num_rows if indices is not None else table.num_rows
+ _check_valid_index_key(key, size)
+ # Query the main table
+ if indices is None:
+ pa_subtable = _query_table(table, key)
+ else:
+ pa_subtable = _query_table_with_indices_mapping(table, key, indices=indices)
+ return pa_subtable
+
+
+def format_table(
+ table: Table,
+ key: Union[int, slice, range, str, Iterable],
+ formatter: Formatter,
+ format_columns: Optional[list] = None,
+ output_all_columns=False,
+):
+ """
+ Format a Table depending on the key that was used and a Formatter object.
+
+ Args:
+ table (``datasets.table.Table``): The input Table to format
+ key (``Union[int, slice, range, str, Iterable]``): Depending on the key that was used, the formatter formats
+ the table as either a row, a column or a batch.
+ formatter (``datasets.formatting.formatting.Formatter``): Any subclass of a Formatter such as
+ PythonFormatter, NumpyFormatter, etc.
+ format_columns (:obj:`List[str]`, optional): if not None, it defines the columns that will be formatted using the
+ given formatter. Other columns are discarded (unless ``output_all_columns`` is True)
+ output_all_columns (:obj:`bool`, defaults to False). If True, the formatted output is completed using the columns
+ that are not in the ``format_columns`` list. For these columns, the PythonFormatter is used.
+
+
+ Returns:
+ A row, column or batch formatted object defined by the Formatter:
+ - the PythonFormatter returns a dictionary for a row or a batch, and a list for a column.
+ - the NumpyFormatter returns a dictionary for a row or a batch, and a np.array for a column.
+ - the PandasFormatter returns a pd.DataFrame for a row or a batch, and a pd.Series for a column.
+ - the TorchFormatter returns a dictionary for a row or a batch, and a torch.Tensor for a column.
+ - the TFFormatter returns a dictionary for a row or a batch, and a tf.Tensor for a column.
+ """
+ if isinstance(table, Table):
+ pa_table = table.table
+ else:
+ pa_table = table
+ query_type = key_to_query_type(key)
+ python_formatter = PythonFormatter(features=formatter.features)
+ if format_columns is None:
+ return formatter(pa_table, query_type=query_type)
+ elif query_type == "column":
+ if key in format_columns:
+ return formatter(pa_table, query_type)
+ else:
+ return python_formatter(pa_table, query_type=query_type)
+ else:
+ pa_table_to_format = pa_table.drop(col for col in pa_table.column_names if col not in format_columns)
+ formatted_output = formatter(pa_table_to_format, query_type=query_type)
+ if output_all_columns:
+ if isinstance(formatted_output, MutableMapping):
+ pa_table_with_remaining_columns = pa_table.drop(
+ col for col in pa_table.column_names if col in format_columns
+ )
+ remaining_columns_dict = python_formatter(pa_table_with_remaining_columns, query_type=query_type)
+ formatted_output.update(remaining_columns_dict)
+ else:
+ raise TypeError(
+ f"Custom formatting function must return a dict to work with output_all_columns=True, but got {formatted_output}"
+ )
+ return formatted_output
diff --git a/venv/Lib/site-packages/datasets/formatting/jax_formatter.py b/venv/Lib/site-packages/datasets/formatting/jax_formatter.py
new file mode 100644
index 0000000000000000000000000000000000000000..8d0cad4e38c9bfd317fde7640ba12d8d3174f0be
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/formatting/jax_formatter.py
@@ -0,0 +1,174 @@
+# Copyright 2021 The HuggingFace Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+import sys
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Optional
+
+import numpy as np
+import pyarrow as pa
+
+from .. import config
+from ..utils.logging import get_logger
+from ..utils.py_utils import map_nested
+from .formatting import TensorFormatter
+
+
+if TYPE_CHECKING:
+ import jax
+ import jaxlib
+
+logger = get_logger()
+
+DEVICE_MAPPING: Optional[dict] = None
+
+
+class JaxFormatter(TensorFormatter[Mapping, "jax.Array", Mapping]):
+ def __init__(self, features=None, device=None, token_per_repo_id=None, **jnp_array_kwargs):
+ super().__init__(features=features, token_per_repo_id=token_per_repo_id)
+ import jax
+ from jaxlib.xla_client import Device
+
+ if isinstance(device, Device):
+ raise ValueError(
+ f"Expected {device} to be a `str` not {type(device)}, as `jaxlib.xla_extension.Device` "
+ "is not serializable neither with `pickle` nor with `dill`. Instead you can surround "
+ "the device with `str()` to get its string identifier that will be internally mapped "
+ "to the actual `jaxlib.xla_extension.Device`."
+ )
+ self.device = device if isinstance(device, str) else str(jax.devices()[0])
+ # using global variable since `jaxlib.xla_extension.Device` is not serializable neither
+ # with `pickle` nor with `dill`, so we need to use a global variable instead
+ global DEVICE_MAPPING
+ if DEVICE_MAPPING is None:
+ DEVICE_MAPPING = self._map_devices_to_str()
+ if self.device not in list(DEVICE_MAPPING.keys()):
+ logger.warning(
+ f"Device with string identifier {self.device} not listed among the available "
+ f"devices: {list(DEVICE_MAPPING.keys())}, so falling back to the default "
+ f"device: {str(jax.devices()[0])}."
+ )
+ self.device = str(jax.devices()[0])
+ self.jnp_array_kwargs = jnp_array_kwargs
+
+ @staticmethod
+ def _map_devices_to_str() -> dict[str, "jaxlib.xla_extension.Device"]:
+ import jax
+
+ return {str(device): device for device in jax.devices()}
+
+ def _consolidate(self, column):
+ import jax
+ import jax.numpy as jnp
+
+ if isinstance(column, list) and column:
+ if all(
+ isinstance(x, jax.Array) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column
+ ):
+ return jnp.stack(column, axis=0)
+ return column
+
+ def _tensorize(self, value):
+ import jax
+ import jax.numpy as jnp
+
+ if isinstance(value, (str, bytes, type(None))):
+ return value
+ elif isinstance(value, (np.character, np.ndarray)) and np.issubdtype(value.dtype, np.character):
+ return value.tolist()
+
+ default_dtype = {}
+
+ if isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.integer):
+ # the default int precision depends on the jax config
+ # see https://jax.readthedocs.io/en/latest/notebooks/Common_Gotchas_in_JAX.html#double-64bit-precision
+ if jax.config.jax_enable_x64:
+ default_dtype = {"dtype": jnp.int64}
+ else:
+ default_dtype = {"dtype": jnp.int32}
+ elif isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.floating):
+ default_dtype = {"dtype": jnp.float32}
+
+ if config.PIL_AVAILABLE and "PIL" in sys.modules:
+ import PIL.Image
+
+ if isinstance(value, PIL.Image.Image):
+ value = np.asarray(value)
+ if config.TORCHVISION_AVAILABLE and "torchvision" in sys.modules:
+ try:
+ from torchvision.io import VideoReader
+
+ if isinstance(value, VideoReader):
+ return value # TODO(QL): set output to jax arrays ?
+ except ImportError:
+ pass
+ if config.TORCHCODEC_AVAILABLE and "torchcodec" in sys.modules:
+ from torchcodec.decoders import AudioDecoder, VideoDecoder
+
+ if isinstance(value, (VideoDecoder, AudioDecoder)):
+ return value # TODO(QL): set output to jax arrays ?
+
+ # using global variable since `jaxlib.xla_extension.Device` is not serializable neither
+ # with `pickle` nor with `dill`, so we need to use a global variable instead
+ global DEVICE_MAPPING
+ if DEVICE_MAPPING is None:
+ DEVICE_MAPPING = self._map_devices_to_str()
+
+ with jax.default_device(DEVICE_MAPPING[self.device]):
+ # calling jnp.array on a np.ndarray does copy the data
+ # see https://github.com/google/jax/issues/4486
+ return jnp.array(value, **{**default_dtype, **self.jnp_array_kwargs})
+
+ def _recursive_tensorize(self, data_struct):
+ import jax
+
+ # support for torch, tf, jax etc.
+ if config.TORCH_AVAILABLE and "torch" in sys.modules:
+ import torch
+
+ if isinstance(data_struct, torch.Tensor):
+ return self._tensorize(data_struct.detach().cpu().numpy()[()])
+ if hasattr(data_struct, "__array__") and not isinstance(data_struct, jax.Array):
+ data_struct = data_struct.__array__()
+ # support for nested types like struct of list of struct
+ if isinstance(data_struct, np.ndarray):
+ if data_struct.dtype == object: # jax arrays cannot be instantied from an array of objects
+ return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct])
+ elif isinstance(data_struct, (list, tuple)):
+ return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct])
+ return self._tensorize(data_struct)
+
+ def recursive_tensorize(self, data_struct: dict):
+ return map_nested(self._recursive_tensorize, data_struct, map_list=False)
+
+ def format_row(self, pa_table: pa.Table) -> Mapping:
+ row = self.numpy_arrow_extractor().extract_row(pa_table)
+ row = self.python_features_decoder.decode_row(row)
+ return self.recursive_tensorize(row)
+
+ def format_column(self, pa_table: pa.Table) -> "jax.Array":
+ column = self.numpy_arrow_extractor().extract_column(pa_table)
+ column = self.python_features_decoder.decode_column(column, pa_table.column_names[0])
+ column = self.recursive_tensorize(column)
+ column = self._consolidate(column)
+ return column
+
+ def format_batch(self, pa_table: pa.Table) -> Mapping:
+ batch = self.numpy_arrow_extractor().extract_batch(pa_table)
+ batch = self.python_features_decoder.decode_batch(batch)
+ batch = self.recursive_tensorize(batch)
+ for column_name in batch:
+ batch[column_name] = self._consolidate(batch[column_name])
+ return batch
diff --git a/venv/Lib/site-packages/datasets/formatting/np_formatter.py b/venv/Lib/site-packages/datasets/formatting/np_formatter.py
new file mode 100644
index 0000000000000000000000000000000000000000..eee14a35eebbd67cc23aa6839600b1852b2d0207
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/formatting/np_formatter.py
@@ -0,0 +1,120 @@
+# Copyright 2020 The HuggingFace Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from collections.abc import Mapping
+
+import numpy as np
+import pyarrow as pa
+
+from .. import config
+from ..utils.py_utils import map_nested
+from .formatting import TensorFormatter
+
+
+class NumpyFormatter(TensorFormatter[Mapping, np.ndarray, Mapping]):
+ def __init__(self, features=None, token_per_repo_id=None, **np_array_kwargs):
+ super().__init__(features=features, token_per_repo_id=token_per_repo_id)
+ self.np_array_kwargs = np_array_kwargs
+
+ def _consolidate(self, column):
+ if isinstance(column, list):
+ if column and all(
+ isinstance(x, np.ndarray) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column
+ ):
+ return np.stack(column)
+ else:
+ # don't use np.array(column, dtype=object)
+ # since it fails in certain cases
+ # see https://stackoverflow.com/q/51005699
+ out = np.empty(len(column), dtype=object)
+ out[:] = column
+ return out
+ return column
+
+ def _tensorize(self, value):
+ if isinstance(value, (str, bytes, type(None))):
+ return value
+ elif isinstance(value, (np.character, np.ndarray)) and np.issubdtype(value.dtype, np.character):
+ return value
+ elif isinstance(value, np.number):
+ return value
+
+ default_dtype = {}
+
+ if isinstance(value, np.ndarray) and np.issubdtype(value.dtype, np.integer):
+ default_dtype = {"dtype": np.int64}
+ elif isinstance(value, np.ndarray) and np.issubdtype(value.dtype, np.floating):
+ default_dtype = {"dtype": np.float32}
+
+ if config.PIL_AVAILABLE and "PIL" in sys.modules:
+ import PIL.Image
+
+ if isinstance(value, PIL.Image.Image):
+ return np.asarray(value, **self.np_array_kwargs)
+ if config.TORCHVISION_AVAILABLE and "torchvision" in sys.modules:
+ try:
+ from torchvision.io import VideoReader
+
+ if isinstance(value, VideoReader):
+ return value # TODO(QL): set output to np arrays ?
+ except ImportError:
+ pass
+ if config.TORCHCODEC_AVAILABLE and "torchcodec" in sys.modules:
+ from torchcodec.decoders import AudioDecoder, VideoDecoder
+
+ if isinstance(value, (VideoDecoder, AudioDecoder)):
+ return value # TODO(QL): set output to np arrays ?
+
+ return np.asarray(value, **{**default_dtype, **self.np_array_kwargs})
+
+ def _recursive_tensorize(self, data_struct):
+ # support for torch, tf, jax etc.
+ if config.TORCH_AVAILABLE and "torch" in sys.modules:
+ import torch
+
+ if isinstance(data_struct, torch.Tensor):
+ return self._tensorize(data_struct.detach().cpu().numpy()[()])
+ if hasattr(data_struct, "__array__") and not isinstance(data_struct, (np.ndarray, np.character, np.number)):
+ data_struct = data_struct.__array__()
+ # support for nested types like struct of list of struct
+ if isinstance(data_struct, np.ndarray):
+ if data_struct.dtype == object:
+ return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct])
+ if isinstance(data_struct, (list, tuple)):
+ return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct])
+ return self._tensorize(data_struct)
+
+ def recursive_tensorize(self, data_struct: dict):
+ return map_nested(self._recursive_tensorize, data_struct, map_list=False)
+
+ def format_row(self, pa_table: pa.Table) -> Mapping:
+ row = self.numpy_arrow_extractor().extract_row(pa_table)
+ row = self.python_features_decoder.decode_row(row)
+ return self.recursive_tensorize(row)
+
+ def format_column(self, pa_table: pa.Table) -> np.ndarray:
+ column = self.numpy_arrow_extractor().extract_column(pa_table)
+ column = self.python_features_decoder.decode_column(column, pa_table.column_names[0])
+ column = self.recursive_tensorize(column)
+ column = self._consolidate(column)
+ return column
+
+ def format_batch(self, pa_table: pa.Table) -> Mapping:
+ batch = self.numpy_arrow_extractor().extract_batch(pa_table)
+ batch = self.python_features_decoder.decode_batch(batch)
+ batch = self.recursive_tensorize(batch)
+ for column_name in batch:
+ batch[column_name] = self._consolidate(batch[column_name])
+ return batch
diff --git a/venv/Lib/site-packages/datasets/formatting/polars_formatter.py b/venv/Lib/site-packages/datasets/formatting/polars_formatter.py
new file mode 100644
index 0000000000000000000000000000000000000000..7ea2f783aec38ac6bbc10d12aff2488655940f3f
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/formatting/polars_formatter.py
@@ -0,0 +1,124 @@
+# Copyright 2020 The HuggingFace Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from functools import partial
+from typing import TYPE_CHECKING, Optional
+
+import pyarrow as pa
+
+from .. import config
+from ..features import Features
+from ..features.features import decode_nested_example
+from ..utils.py_utils import no_op_if_value_is_null
+from .formatting import BaseArrowExtractor, TableFormatter
+
+
+if TYPE_CHECKING:
+ import polars as pl
+
+
+class PolarsArrowExtractor(BaseArrowExtractor["pl.DataFrame", "pl.Series", "pl.DataFrame"]):
+ def extract_row(self, pa_table: pa.Table) -> "pl.DataFrame":
+ if config.POLARS_AVAILABLE:
+ if "polars" not in sys.modules:
+ import polars
+ else:
+ polars = sys.modules["polars"]
+
+ return polars.from_arrow(pa_table.slice(length=1))
+ else:
+ raise ValueError("Polars needs to be installed to be able to return Polars dataframes.")
+
+ def extract_column(self, pa_table: pa.Table) -> "pl.Series":
+ if config.POLARS_AVAILABLE:
+ if "polars" not in sys.modules:
+ import polars
+ else:
+ polars = sys.modules["polars"]
+
+ return polars.from_arrow(pa_table.select([0]))[pa_table.column_names[0]]
+ else:
+ raise ValueError("Polars needs to be installed to be able to return Polars dataframes.")
+
+ def extract_batch(self, pa_table: pa.Table) -> "pl.DataFrame":
+ if config.POLARS_AVAILABLE:
+ if "polars" not in sys.modules:
+ import polars
+ else:
+ polars = sys.modules["polars"]
+
+ return polars.from_arrow(pa_table)
+ else:
+ raise ValueError("Polars needs to be installed to be able to return Polars dataframes.")
+
+
+class PolarsFeaturesDecoder:
+ def __init__(self, features: Optional[Features]):
+ self.features = features
+ import polars as pl # noqa: F401 - import pl at initialization
+
+ def decode_row(self, row: "pl.DataFrame") -> "pl.DataFrame":
+ decode = (
+ {
+ column_name: no_op_if_value_is_null(partial(decode_nested_example, feature))
+ for column_name, feature in self.features.items()
+ if self.features._column_requires_decoding[column_name]
+ }
+ if self.features
+ else {}
+ )
+ if decode:
+ row[list(decode.keys())] = row.map_rows(decode)
+ return row
+
+ def decode_column(self, column: "pl.Series", column_name: str) -> "pl.Series":
+ decode = (
+ no_op_if_value_is_null(partial(decode_nested_example, self.features[column_name]))
+ if self.features and column_name in self.features and self.features._column_requires_decoding[column_name]
+ else None
+ )
+ if decode:
+ column = column.map_elements(decode)
+ return column
+
+ def decode_batch(self, batch: "pl.DataFrame") -> "pl.DataFrame":
+ return self.decode_row(batch)
+
+
+class PolarsFormatter(TableFormatter["pl.DataFrame", "pl.Series", "pl.DataFrame"]):
+ table_type = "polars dataframe"
+ column_type = "polars series"
+
+ def __init__(self, features=None, **np_array_kwargs):
+ super().__init__(features=features)
+ self.np_array_kwargs = np_array_kwargs
+ self.polars_arrow_extractor = PolarsArrowExtractor
+ self.polars_features_decoder = PolarsFeaturesDecoder(features)
+ import polars as pl # noqa: F401 - import pl at initialization
+
+ def format_row(self, pa_table: pa.Table) -> "pl.DataFrame":
+ row = self.polars_arrow_extractor().extract_row(pa_table)
+ row = self.polars_features_decoder.decode_row(row)
+ return row
+
+ def format_column(self, pa_table: pa.Table) -> "pl.Series":
+ column = self.polars_arrow_extractor().extract_column(pa_table)
+ column = self.polars_features_decoder.decode_column(column, pa_table.column_names[0])
+ return column
+
+ def format_batch(self, pa_table: pa.Table) -> "pl.DataFrame":
+ row = self.polars_arrow_extractor().extract_batch(pa_table)
+ row = self.polars_features_decoder.decode_batch(row)
+ return row
diff --git a/venv/Lib/site-packages/datasets/formatting/tf_formatter.py b/venv/Lib/site-packages/datasets/formatting/tf_formatter.py
new file mode 100644
index 0000000000000000000000000000000000000000..028b998d5d1afd17ef1705930a8d6bb02e7c9ad8
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/formatting/tf_formatter.py
@@ -0,0 +1,129 @@
+# Copyright 2020 The HuggingFace Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+import sys
+from collections.abc import Mapping
+from typing import TYPE_CHECKING
+
+import numpy as np
+import pyarrow as pa
+
+from .. import config
+from ..utils.py_utils import map_nested
+from .formatting import TensorFormatter
+
+
+if TYPE_CHECKING:
+ import tensorflow as tf
+
+
+class TFFormatter(TensorFormatter[Mapping, "tf.Tensor", Mapping]):
+ def __init__(self, features=None, token_per_repo_id=None, **tf_tensor_kwargs):
+ super().__init__(features=features, token_per_repo_id=token_per_repo_id)
+ self.tf_tensor_kwargs = tf_tensor_kwargs
+ import tensorflow as tf # noqa: F401 - import tf at initialization
+
+ def _consolidate(self, column):
+ import tensorflow as tf
+
+ if isinstance(column, list) and column:
+ if all(
+ isinstance(x, tf.Tensor) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column
+ ):
+ return tf.stack(column)
+ elif all(
+ isinstance(x, (tf.Tensor, tf.RaggedTensor)) and x.ndim == 1 and x.dtype == column[0].dtype
+ for x in column
+ ):
+ # only rag 1-D tensors, otherwise some dimensions become ragged even though they were consolidated
+ return tf.ragged.stack(column)
+
+ return column
+
+ def _tensorize(self, value):
+ import tensorflow as tf
+
+ if value is None:
+ return value
+
+ default_dtype = {}
+
+ if isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.integer):
+ default_dtype = {"dtype": tf.int64}
+ elif isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.floating):
+ default_dtype = {"dtype": tf.float32}
+
+ if config.PIL_AVAILABLE and "PIL" in sys.modules:
+ import PIL.Image
+
+ if isinstance(value, PIL.Image.Image):
+ value = np.asarray(value)
+ if config.TORCHVISION_AVAILABLE and "torchvision" in sys.modules:
+ try:
+ from torchvision.io import VideoReader
+
+ if isinstance(value, VideoReader):
+ return value # TODO(QL): set output to tf tensors ?
+ except ImportError:
+ pass
+ if config.TORCHCODEC_AVAILABLE and "torchcodec" in sys.modules:
+ from torchcodec.decoders import AudioDecoder, VideoDecoder
+
+ if isinstance(value, (VideoDecoder, AudioDecoder)):
+ return value # TODO(QL): set output to jax arrays ?
+
+ return tf.convert_to_tensor(value, **{**default_dtype, **self.tf_tensor_kwargs})
+
+ def _recursive_tensorize(self, data_struct):
+ import tensorflow as tf
+
+ # support for torch, tf, jax etc.
+ if config.TORCH_AVAILABLE and "torch" in sys.modules:
+ import torch
+
+ if isinstance(data_struct, torch.Tensor):
+ return self._tensorize(data_struct.detach().cpu().numpy()[()])
+ if hasattr(data_struct, "__array__") and not isinstance(data_struct, tf.Tensor):
+ data_struct = data_struct.__array__()
+ # support for nested types like struct of list of struct
+ if isinstance(data_struct, np.ndarray):
+ if data_struct.dtype == object: # tf tensors cannot be instantied from an array of objects
+ return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct])
+ elif isinstance(data_struct, (list, tuple)):
+ return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct])
+ return self._tensorize(data_struct)
+
+ def recursive_tensorize(self, data_struct: dict):
+ return map_nested(self._recursive_tensorize, data_struct, map_list=False)
+
+ def format_row(self, pa_table: pa.Table) -> Mapping:
+ row = self.numpy_arrow_extractor().extract_row(pa_table)
+ row = self.python_features_decoder.decode_row(row)
+ return self.recursive_tensorize(row)
+
+ def format_column(self, pa_table: pa.Table) -> "tf.Tensor":
+ column = self.numpy_arrow_extractor().extract_column(pa_table)
+ column = self.python_features_decoder.decode_column(column, pa_table.column_names[0])
+ column = self.recursive_tensorize(column)
+ column = self._consolidate(column)
+ return column
+
+ def format_batch(self, pa_table: pa.Table) -> Mapping:
+ batch = self.numpy_arrow_extractor().extract_batch(pa_table)
+ batch = self.python_features_decoder.decode_batch(batch)
+ batch = self.recursive_tensorize(batch)
+ for column_name in batch:
+ batch[column_name] = self._consolidate(batch[column_name])
+ return batch
diff --git a/venv/Lib/site-packages/datasets/formatting/torch_formatter.py b/venv/Lib/site-packages/datasets/formatting/torch_formatter.py
new file mode 100644
index 0000000000000000000000000000000000000000..a41dac9ed66909f296dd12ec5b751ed72b739257
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/formatting/torch_formatter.py
@@ -0,0 +1,130 @@
+# Copyright 2020 The HuggingFace Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+import sys
+from collections.abc import Mapping
+from typing import TYPE_CHECKING
+
+import numpy as np
+import pyarrow as pa
+
+from .. import config
+from ..utils.py_utils import map_nested
+from .formatting import TensorFormatter
+
+
+if TYPE_CHECKING:
+ import torch
+
+
+class TorchFormatter(TensorFormatter[Mapping, "torch.Tensor", Mapping]):
+ def __init__(self, features=None, token_per_repo_id=None, **torch_tensor_kwargs):
+ super().__init__(features=features, token_per_repo_id=token_per_repo_id)
+ self.torch_tensor_kwargs = torch_tensor_kwargs
+ import torch # noqa import torch at initialization
+
+ def _consolidate(self, column):
+ import torch
+
+ if isinstance(column, list) and column:
+ if all(
+ isinstance(x, torch.Tensor) and x.shape == column[0].shape and x.dtype == column[0].dtype
+ for x in column
+ ):
+ return torch.stack(column)
+ return column
+
+ def _tensorize(self, value):
+ import torch
+
+ if isinstance(value, (str, bytes, type(None))):
+ return value
+ elif isinstance(value, (np.character, np.ndarray)) and np.issubdtype(value.dtype, np.character):
+ return value.tolist()
+
+ default_dtype = {}
+
+ if isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.integer):
+ default_dtype = {"dtype": torch.int64}
+
+ # Convert dtype to np.int64 if it's either np.uint16 or np.uint32 to ensure compatibility.
+ # np.uint64 is excluded from this conversion as there is no compatible PyTorch dtype that can handle it without loss.
+ if value.dtype in [np.uint16, np.uint32]:
+ value = value.astype(np.int64)
+
+ elif isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.floating):
+ default_dtype = {"dtype": torch.float32}
+
+ if config.PIL_AVAILABLE and "PIL" in sys.modules:
+ import PIL.Image
+
+ if isinstance(value, PIL.Image.Image):
+ value = np.asarray(value)
+ if value.ndim == 2:
+ value = value[:, :, np.newaxis]
+
+ value = value.transpose((2, 0, 1))
+ if config.TORCHVISION_AVAILABLE and "torchvision" in sys.modules:
+ try:
+ from torchvision.io import VideoReader
+
+ if isinstance(value, VideoReader):
+ return value # TODO(QL): set output to torch tensors ?
+ except ImportError:
+ pass
+ if config.TORCHCODEC_AVAILABLE and "torchcodec" in sys.modules:
+ from torchcodec.decoders import AudioDecoder, VideoDecoder
+
+ if isinstance(value, (VideoDecoder, AudioDecoder)):
+ return value # TODO(QL): set output to jax arrays ?
+
+ return torch.tensor(value, **{**default_dtype, **self.torch_tensor_kwargs})
+
+ def _recursive_tensorize(self, data_struct):
+ import torch
+
+ # support for torch, tf, jax etc.
+ if hasattr(data_struct, "__array__") and not isinstance(data_struct, torch.Tensor):
+ data_struct = data_struct.__array__()
+ # support for nested types like struct of list of struct
+ if isinstance(data_struct, np.ndarray):
+ if data_struct.dtype == object: # torch tensors cannot be instantied from an array of objects
+ return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct])
+ elif isinstance(data_struct, (list, tuple)):
+ return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct])
+ return self._tensorize(data_struct)
+
+ def recursive_tensorize(self, data_struct: dict):
+ return map_nested(self._recursive_tensorize, data_struct, map_list=False)
+
+ def format_row(self, pa_table: pa.Table) -> Mapping:
+ row = self.numpy_arrow_extractor().extract_row(pa_table)
+ row = self.python_features_decoder.decode_row(row)
+ return self.recursive_tensorize(row)
+
+ def format_column(self, pa_table: pa.Table) -> "torch.Tensor":
+ column = self.numpy_arrow_extractor().extract_column(pa_table)
+ column = self.python_features_decoder.decode_column(column, pa_table.column_names[0])
+ column = self.recursive_tensorize(column)
+ column = self._consolidate(column)
+ return column
+
+ def format_batch(self, pa_table: pa.Table) -> Mapping:
+ batch = self.numpy_arrow_extractor().extract_batch(pa_table)
+ batch = self.python_features_decoder.decode_batch(batch)
+ batch = self.recursive_tensorize(batch)
+ for column_name in batch:
+ batch[column_name] = self._consolidate(batch[column_name])
+ return batch
diff --git a/venv/Lib/site-packages/datasets/hub.py b/venv/Lib/site-packages/datasets/hub.py
new file mode 100644
index 0000000000000000000000000000000000000000..fd64e04cdee49560063cf76dd4422780296c86c1
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/hub.py
@@ -0,0 +1,135 @@
+from itertools import chain
+from typing import Optional, Union
+
+from huggingface_hub import (
+ CommitInfo,
+ CommitOperationAdd,
+ CommitOperationDelete,
+ DatasetCard,
+ DatasetCardData,
+ HfApi,
+ HfFileSystem,
+)
+
+import datasets.config
+from datasets import __version__
+from datasets.info import DatasetInfosDict
+from datasets.load import load_dataset_builder
+from datasets.utils.metadata import MetadataConfigs
+
+
+def delete_from_hub(
+ repo_id: str,
+ config_name: str,
+ revision: Optional[str] = None,
+ token: Optional[Union[bool, str]] = None,
+) -> CommitInfo:
+ """Delete a dataset configuration from a [data-only dataset](repository_structure) on the Hub.
+
+ Args:
+ repo_id (`str`): ID of the Hub dataset repository, in the following format: `/` or
+ `/`.
+ config_name (`str`): Name of the dataset configuration.
+ revision (`str`, *optional*): Branch to delete the configuration from. Defaults to the `"main"` branch.
+ token (`bool` or `str`, *optional*): Authentication token for the Hugging Face Hub.
+
+ Returns:
+ `huggingface_hub.CommitInfo`
+ """
+ operations = []
+ # data_files
+ fs = HfFileSystem(endpoint=datasets.config.HF_ENDPOINT, token=token)
+ builder = load_dataset_builder(repo_id, config_name, revision=revision, token=token)
+ for data_file in chain(*builder.config.data_files.values()):
+ data_file_resolved_path = fs.resolve_path(data_file)
+ if data_file_resolved_path.repo_id == repo_id:
+ operations.append(CommitOperationDelete(path_in_repo=data_file_resolved_path.path_in_repo))
+ # README.md
+ dataset_card = DatasetCard.load(repo_id)
+ # config_names
+ if dataset_card.data.get("config_names", None) and config_name in dataset_card.data["config_names"]:
+ dataset_card.data["config_names"].remove(config_name)
+ # metadata_configs
+ metadata_configs = MetadataConfigs.from_dataset_card_data(dataset_card.data)
+ if metadata_configs:
+ _ = metadata_configs.pop(config_name, None)
+ dataset_card_data = DatasetCardData()
+ metadata_configs.to_dataset_card_data(dataset_card_data)
+ if datasets.config.METADATA_CONFIGS_FIELD in dataset_card_data:
+ dataset_card.data[datasets.config.METADATA_CONFIGS_FIELD] = dataset_card_data[
+ datasets.config.METADATA_CONFIGS_FIELD
+ ]
+ else:
+ _ = dataset_card.data.pop(datasets.config.METADATA_CONFIGS_FIELD, None)
+ # dataset_info
+ dataset_infos: DatasetInfosDict = DatasetInfosDict.from_dataset_card_data(dataset_card.data)
+ if dataset_infos:
+ _ = dataset_infos.pop(config_name, None)
+ dataset_card_data = DatasetCardData()
+ dataset_infos.to_dataset_card_data(dataset_card_data)
+ if "dataset_info" in dataset_card_data:
+ dataset_card.data["dataset_info"] = dataset_card_data["dataset_info"]
+ else:
+ _ = dataset_card.data.pop("dataset_info", None)
+ # Commit
+ operations.append(
+ CommitOperationAdd(path_in_repo=datasets.config.REPOCARD_FILENAME, path_or_fileobj=str(dataset_card).encode())
+ )
+ api = HfApi(
+ endpoint=datasets.config.HF_ENDPOINT,
+ token=token,
+ library_name="datasets",
+ library_version=__version__,
+ )
+ commit_info = api.create_commit(
+ repo_id,
+ operations=operations,
+ commit_message=f"Delete '{config_name}' config",
+ commit_description=f"Delete '{config_name}' config.",
+ token=token,
+ repo_type="dataset",
+ revision=revision,
+ create_pr=True,
+ )
+ print(f"You can find your PR to delete the dataset config at: {commit_info.pr_url}")
+ return commit_info
+
+
+def _delete_files(dataset_id, revision=None, token=None):
+ hf_api = HfApi(
+ endpoint=datasets.config.HF_ENDPOINT,
+ token=token,
+ library_name="datasets",
+ library_version=__version__,
+ )
+ repo_files = hf_api.list_repo_files(
+ dataset_id,
+ repo_type="dataset",
+ )
+ if repo_files:
+ legacy_json_file = []
+ data_files = []
+ for filename in repo_files:
+ if filename in {".gitattributes", "README.md"}:
+ continue
+ elif filename == "dataset_infos.json":
+ legacy_json_file.append(filename)
+ else:
+ data_files.append(filename)
+ if legacy_json_file:
+ hf_api.delete_file(
+ "dataset_infos.json",
+ dataset_id,
+ repo_type="dataset",
+ revision=revision,
+ commit_message="Delete legacy dataset_infos.json",
+ )
+ if data_files:
+ for filename in data_files:
+ hf_api.delete_file(
+ filename,
+ dataset_id,
+ repo_type="dataset",
+ revision=revision,
+ commit_message="Delete data file",
+ )
diff --git a/venv/Lib/site-packages/datasets/info.py b/venv/Lib/site-packages/datasets/info.py
new file mode 100644
index 0000000000000000000000000000000000000000..13fd972cb1a57f47a8ff581d7937f7a3411698a7
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/info.py
@@ -0,0 +1,440 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""DatasetInfo record information we know about a dataset.
+
+This includes things that we know about the dataset statically, i.e.:
+ - description
+ - canonical location
+ - does it have validation and tests splits
+ - size
+ - etc.
+
+This also includes the things that can and should be computed once we've
+processed the dataset as well:
+ - number of examples (in each split)
+ - etc.
+"""
+
+import copy
+import dataclasses
+import json
+import os
+import posixpath
+from dataclasses import dataclass
+from pathlib import Path
+from typing import ClassVar, Optional, Union
+
+import fsspec
+from fsspec.core import url_to_fs
+from huggingface_hub import DatasetCard, DatasetCardData
+
+from . import config
+from .features import Features
+from .splits import SplitDict
+from .utils import Version
+from .utils.logging import get_logger
+from .utils.py_utils import asdict, unique_values
+
+
+logger = get_logger(__name__)
+
+
+@dataclass
+class SupervisedKeysData:
+ input: str = ""
+ output: str = ""
+
+
+@dataclass
+class DownloadChecksumsEntryData:
+ key: str = ""
+ value: str = ""
+
+
+class MissingCachedSizesConfigError(Exception):
+ """The expected cached sizes of the download file are missing."""
+
+
+class NonMatchingCachedSizesError(Exception):
+ """The prepared split doesn't have expected sizes."""
+
+
+@dataclass
+class PostProcessedInfo:
+ features: Optional[Features] = None
+ resources_checksums: Optional[dict] = None
+
+ def __post_init__(self):
+ # Convert back to the correct classes when we reload from dict
+ if self.features is not None and not isinstance(self.features, Features):
+ self.features = Features.from_dict(self.features)
+
+ @classmethod
+ def from_dict(cls, post_processed_info_dict: dict) -> "PostProcessedInfo":
+ field_names = {f.name for f in dataclasses.fields(cls)}
+ return cls(**{k: v for k, v in post_processed_info_dict.items() if k in field_names})
+
+
+@dataclass
+class DatasetInfo:
+ """Information about a dataset.
+
+ `DatasetInfo` documents datasets, including its name, version, and features.
+ See the constructor arguments and properties for a full list.
+
+ Not all fields are known on construction and may be updated later.
+
+ Attributes:
+ description (`str`):
+ A description of the dataset.
+ citation (`str`):
+ A BibTeX citation of the dataset.
+ homepage (`str`):
+ A URL to the official homepage for the dataset.
+ license (`str`):
+ The dataset's license. It can be the name of the license or a paragraph containing the terms of the license.
+ features ([`Features`], *optional*):
+ The features used to specify the dataset's column types.
+ post_processed (`PostProcessedInfo`, *optional*):
+ Deprecated. Information regarding the resources of a possible post-processing of a dataset. For example, it can contain the information of an index.
+ supervised_keys (`SupervisedKeysData`, *optional*):
+ Specifies the input feature and the label for supervised learning if applicable for the dataset (legacy from TFDS).
+ builder_name (`str`, *optional*):
+ The name of the `GeneratorBasedBuilder` subclass used to create the dataset. It is also the snake_case version of the dataset builder class name.
+ config_name (`str`, *optional*):
+ The name of the configuration derived from [`BuilderConfig`].
+ version (`str` or [`Version`], *optional*):
+ The version of the dataset.
+ splits (`dict`, *optional*):
+ The mapping between split name and metadata.
+ download_checksums (`dict`, *optional*):
+ The mapping between the URL to download the dataset's checksums and corresponding metadata.
+ download_size (`int`, *optional*):
+ The size of the files to download to generate the dataset, in bytes.
+ post_processing_size (`int`, *optional*):
+ Deprecated. Size of the dataset in bytes after post-processing, if any.
+ dataset_size (`int`, *optional*):
+ The combined size in bytes of the Arrow tables for all splits.
+ size_in_bytes (`int`, *optional*):
+ The combined size in bytes of all files associated with the dataset (downloaded files + Arrow files).
+ **config_kwargs (additional keyword arguments):
+ Keyword arguments to be passed to the [`BuilderConfig`] and used in the [`DatasetBuilder`].
+ """
+
+ # Set in the dataset builders
+ description: str = dataclasses.field(default_factory=str)
+ citation: str = dataclasses.field(default_factory=str)
+ homepage: str = dataclasses.field(default_factory=str)
+ license: str = dataclasses.field(default_factory=str)
+ features: Optional[Features] = None
+ post_processed: Optional[PostProcessedInfo] = None # kept for bawkard compat
+ supervised_keys: Optional[SupervisedKeysData] = None
+
+ # Set later by the builder
+ builder_name: Optional[str] = None
+ dataset_name: Optional[str] = None # for packaged builders, to be different from builder_name
+ config_name: Optional[str] = None
+ version: Optional[Union[str, Version]] = None
+ # Set later by `download_and_prepare`
+ splits: Optional[SplitDict] = None
+ download_checksums: Optional[dict] = None
+ download_size: Optional[int] = None
+ post_processing_size: Optional[int] = None
+ dataset_size: Optional[int] = None
+ size_in_bytes: Optional[int] = None
+
+ _INCLUDED_INFO_IN_YAML: ClassVar[list[str]] = [
+ "config_name",
+ "download_size",
+ "dataset_size",
+ "features",
+ "splits",
+ ]
+
+ def __post_init__(self):
+ # Convert back to the correct classes when we reload from dict
+ if self.features is not None and not isinstance(self.features, Features):
+ self.features = Features.from_dict(self.features)
+ if self.post_processed is not None and not isinstance(self.post_processed, PostProcessedInfo):
+ self.post_processed = PostProcessedInfo.from_dict(self.post_processed)
+ if self.version is not None and not isinstance(self.version, Version):
+ if isinstance(self.version, str):
+ self.version = Version(self.version)
+ else:
+ self.version = Version.from_dict(self.version)
+ if self.splits is not None and not isinstance(self.splits, SplitDict):
+ self.splits = SplitDict.from_split_dict(self.splits)
+ if self.supervised_keys is not None and not isinstance(self.supervised_keys, SupervisedKeysData):
+ if isinstance(self.supervised_keys, (tuple, list)):
+ self.supervised_keys = SupervisedKeysData(*self.supervised_keys)
+ else:
+ self.supervised_keys = SupervisedKeysData(**self.supervised_keys)
+
+ def write_to_directory(self, dataset_info_dir, pretty_print=False, storage_options: Optional[dict] = None):
+ """Write `DatasetInfo` and license (if present) as JSON files to `dataset_info_dir`.
+
+ Args:
+ dataset_info_dir (`str`):
+ Destination directory.
+ pretty_print (`bool`, defaults to `False`):
+ If `True`, the JSON will be pretty-printed with the indent level of 4.
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+
+
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds.info.write_to_directory("/path/to/directory/")
+ ```
+ """
+ fs: fsspec.AbstractFileSystem
+ fs, *_ = url_to_fs(dataset_info_dir, **(storage_options or {}))
+ with fs.open(posixpath.join(dataset_info_dir, config.DATASET_INFO_FILENAME), "wb") as f:
+ self._dump_info(f, pretty_print=pretty_print)
+ if self.license:
+ with fs.open(posixpath.join(dataset_info_dir, config.LICENSE_FILENAME), "wb") as f:
+ self._dump_license(f)
+
+ def _dump_info(self, file, pretty_print=False):
+ """Dump info in `file` file-like object open in bytes mode (to support remote files)"""
+ file.write(json.dumps(asdict(self), indent=4 if pretty_print else None).encode("utf-8"))
+
+ def _dump_license(self, file):
+ """Dump license in `file` file-like object open in bytes mode (to support remote files)"""
+ file.write(self.license.encode("utf-8"))
+
+ @classmethod
+ def from_merge(cls, dataset_infos: list["DatasetInfo"]):
+ dataset_infos = [dset_info.copy() for dset_info in dataset_infos if dset_info is not None]
+
+ if len(dataset_infos) > 0 and all(dataset_infos[0] == dset_info for dset_info in dataset_infos):
+ # if all dataset_infos are equal we don't need to merge. Just return the first.
+ return dataset_infos[0]
+
+ description = "\n\n".join(unique_values(info.description for info in dataset_infos)).strip()
+ citation = "\n\n".join(unique_values(info.citation for info in dataset_infos)).strip()
+ homepage = "\n\n".join(unique_values(info.homepage for info in dataset_infos)).strip()
+ license = "\n\n".join(unique_values(info.license for info in dataset_infos)).strip()
+ features = None
+ supervised_keys = None
+
+ return cls(
+ description=description,
+ citation=citation,
+ homepage=homepage,
+ license=license,
+ features=features,
+ supervised_keys=supervised_keys,
+ )
+
+ @classmethod
+ def from_directory(cls, dataset_info_dir: str, storage_options: Optional[dict] = None) -> "DatasetInfo":
+ """Create [`DatasetInfo`] from the JSON file in `dataset_info_dir`.
+
+ This function updates all the dynamically generated fields (num_examples,
+ hash, time of creation,...) of the [`DatasetInfo`].
+
+ This will overwrite all previous metadata.
+
+ Args:
+ dataset_info_dir (`str`):
+ The directory containing the metadata file. This
+ should be the root directory of a specific dataset version.
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+
+
+
+ Example:
+
+ ```py
+ >>> from datasets import DatasetInfo
+ >>> ds_info = DatasetInfo.from_directory("/path/to/directory/")
+ ```
+ """
+ fs: fsspec.AbstractFileSystem
+ fs, *_ = url_to_fs(dataset_info_dir, **(storage_options or {}))
+ logger.debug(f"Loading Dataset info from {dataset_info_dir}")
+ if not dataset_info_dir:
+ raise ValueError("Calling DatasetInfo.from_directory() with undefined dataset_info_dir.")
+ with fs.open(posixpath.join(dataset_info_dir, config.DATASET_INFO_FILENAME), "r", encoding="utf-8") as f:
+ dataset_info_dict = json.load(f)
+ return cls.from_dict(dataset_info_dict)
+
+ @classmethod
+ def from_dict(cls, dataset_info_dict: dict) -> "DatasetInfo":
+ field_names = {f.name for f in dataclasses.fields(cls)}
+ return cls(**{k: v for k, v in dataset_info_dict.items() if k in field_names})
+
+ def update(self, other_dataset_info: "DatasetInfo", ignore_none=True):
+ self_dict = self.__dict__
+ self_dict.update(
+ **{
+ k: copy.deepcopy(v)
+ for k, v in other_dataset_info.__dict__.items()
+ if (v is not None or not ignore_none)
+ }
+ )
+
+ def copy(self) -> "DatasetInfo":
+ return self.__class__(**{k: copy.deepcopy(v) for k, v in self.__dict__.items()})
+
+ def _to_yaml_dict(self) -> dict:
+ yaml_dict = {}
+ dataset_info_dict = asdict(self)
+ for key in dataset_info_dict:
+ if key in self._INCLUDED_INFO_IN_YAML:
+ value = getattr(self, key)
+ if hasattr(value, "_to_yaml_list"): # Features, SplitDict
+ yaml_dict[key] = value._to_yaml_list()
+ elif hasattr(value, "_to_yaml_string"): # Version
+ yaml_dict[key] = value._to_yaml_string()
+ else:
+ yaml_dict[key] = value
+ return yaml_dict
+
+ @classmethod
+ def _from_yaml_dict(cls, yaml_data: dict) -> "DatasetInfo":
+ yaml_data = copy.deepcopy(yaml_data)
+ if yaml_data.get("features") is not None:
+ yaml_data["features"] = Features._from_yaml_list(yaml_data["features"])
+ if yaml_data.get("splits") is not None:
+ yaml_data["splits"] = SplitDict._from_yaml_list(yaml_data["splits"])
+ field_names = {f.name for f in dataclasses.fields(cls)}
+ return cls(**{k: v for k, v in yaml_data.items() if k in field_names})
+
+ def __repr__(self):
+ return (
+ self.__class__.__qualname__
+ + "("
+ + ", ".join(
+ [f"{f.name}={repr(getattr(self, f.name))}" for f in dataclasses.fields(self) if getattr(self, f.name)]
+ )
+ + ")"
+ )
+
+
+class DatasetInfosDict(dict[str, DatasetInfo]):
+ def write_to_directory(self, dataset_infos_dir, overwrite=False, pretty_print=False) -> None:
+ total_dataset_infos = {}
+ dataset_infos_path = os.path.join(dataset_infos_dir, config.DATASETDICT_INFOS_FILENAME)
+ dataset_readme_path = os.path.join(dataset_infos_dir, config.REPOCARD_FILENAME)
+ if not overwrite:
+ total_dataset_infos = self.from_directory(dataset_infos_dir)
+ total_dataset_infos.update(self)
+ if os.path.exists(dataset_infos_path):
+ # for backward compatibility, let's update the JSON file if it exists
+ with open(dataset_infos_path, "w", encoding="utf-8") as f:
+ dataset_infos_dict = {
+ config_name: asdict(dset_info) for config_name, dset_info in total_dataset_infos.items()
+ }
+ json.dump(dataset_infos_dict, f, indent=4 if pretty_print else None)
+ # Dump the infos in the YAML part of the README.md file
+ if os.path.exists(dataset_readme_path):
+ dataset_card = DatasetCard.load(dataset_readme_path)
+ dataset_card_data = dataset_card.data
+ else:
+ dataset_card = None
+ dataset_card_data = DatasetCardData()
+ if total_dataset_infos:
+ total_dataset_infos.to_dataset_card_data(dataset_card_data)
+ dataset_card = (
+ DatasetCard("---\n" + str(dataset_card_data) + "\n---\n") if dataset_card is None else dataset_card
+ )
+ dataset_card.save(Path(dataset_readme_path))
+
+ @classmethod
+ def from_directory(cls, dataset_infos_dir) -> "DatasetInfosDict":
+ logger.debug(f"Loading Dataset Infos from {dataset_infos_dir}")
+ # Load the info from the YAML part of README.md
+ if os.path.exists(os.path.join(dataset_infos_dir, config.REPOCARD_FILENAME)):
+ dataset_card_data = DatasetCard.load(Path(dataset_infos_dir) / config.REPOCARD_FILENAME).data
+ if "dataset_info" in dataset_card_data:
+ return cls.from_dataset_card_data(dataset_card_data)
+ if os.path.exists(os.path.join(dataset_infos_dir, config.DATASETDICT_INFOS_FILENAME)):
+ # this is just to have backward compatibility with dataset_infos.json files
+ with open(os.path.join(dataset_infos_dir, config.DATASETDICT_INFOS_FILENAME), encoding="utf-8") as f:
+ return cls(
+ {
+ config_name: DatasetInfo.from_dict(dataset_info_dict)
+ for config_name, dataset_info_dict in json.load(f).items()
+ }
+ )
+ else:
+ return cls()
+
+ @classmethod
+ def from_dataset_card_data(cls, dataset_card_data: DatasetCardData) -> "DatasetInfosDict":
+ if isinstance(dataset_card_data.get("dataset_info"), (list, dict)):
+ if isinstance(dataset_card_data["dataset_info"], list):
+ return cls(
+ {
+ dataset_info_yaml_dict.get("config_name", "default"): DatasetInfo._from_yaml_dict(
+ dataset_info_yaml_dict
+ )
+ for dataset_info_yaml_dict in dataset_card_data["dataset_info"]
+ }
+ )
+ else:
+ dataset_info = DatasetInfo._from_yaml_dict(dataset_card_data["dataset_info"])
+ dataset_info.config_name = dataset_card_data["dataset_info"].get("config_name", "default")
+ return cls({dataset_info.config_name: dataset_info})
+ else:
+ return cls()
+
+ def to_dataset_card_data(self, dataset_card_data: DatasetCardData) -> None:
+ if self:
+ # first get existing metadata info
+ if "dataset_info" in dataset_card_data and isinstance(dataset_card_data["dataset_info"], dict):
+ dataset_metadata_infos = {
+ dataset_card_data["dataset_info"].get("config_name", "default"): dataset_card_data["dataset_info"]
+ }
+ elif "dataset_info" in dataset_card_data and isinstance(dataset_card_data["dataset_info"], list):
+ dataset_metadata_infos = {
+ config_metadata["config_name"]: config_metadata
+ for config_metadata in dataset_card_data["dataset_info"]
+ }
+ else:
+ dataset_metadata_infos = {}
+ # update/rewrite existing metadata info with the one to dump
+ total_dataset_infos = {
+ **dataset_metadata_infos,
+ **{config_name: dset_info._to_yaml_dict() for config_name, dset_info in self.items()},
+ }
+ # the config_name from the dataset_infos_dict takes over the config_name of the DatasetInfo
+ for config_name, dset_info_yaml_dict in total_dataset_infos.items():
+ dset_info_yaml_dict["config_name"] = config_name
+ if len(total_dataset_infos) == 1:
+ # use a struct instead of a list of configurations, since there's only one
+ dataset_card_data["dataset_info"] = next(iter(total_dataset_infos.values()))
+ config_name = dataset_card_data["dataset_info"].pop("config_name", None)
+ if config_name != "default":
+ # if config_name is not "default" preserve it and put at the first position
+ dataset_card_data["dataset_info"] = {
+ "config_name": config_name,
+ **dataset_card_data["dataset_info"],
+ }
+ else:
+ dataset_card_data["dataset_info"] = []
+ for config_name, dataset_info_yaml_dict in sorted(total_dataset_infos.items()):
+ # add the config_name field in first position
+ dataset_info_yaml_dict.pop("config_name", None)
+ dataset_info_yaml_dict = {"config_name": config_name, **dataset_info_yaml_dict}
+ dataset_card_data["dataset_info"].append(dataset_info_yaml_dict)
diff --git a/venv/Lib/site-packages/datasets/inspect.py b/venv/Lib/site-packages/datasets/inspect.py
new file mode 100644
index 0000000000000000000000000000000000000000..f07c825d02c2c04112579631db84d93ef2de5c28
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/inspect.py
@@ -0,0 +1,350 @@
+# Copyright 2020 The HuggingFace Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""List and inspect datasets."""
+
+import os
+from collections.abc import Mapping, Sequence
+from typing import Optional, Union
+
+from .download.download_config import DownloadConfig
+from .download.download_manager import DownloadMode
+from .download.streaming_download_manager import StreamingDownloadManager
+from .info import DatasetInfo
+from .load import (
+ dataset_module_factory,
+ get_dataset_builder_class,
+ load_dataset_builder,
+)
+from .utils.logging import get_logger
+from .utils.version import Version
+
+
+logger = get_logger(__name__)
+
+
+class SplitsNotFoundError(ValueError):
+ pass
+
+
+def get_dataset_infos(
+ path: str,
+ data_files: Optional[Union[dict, list, str]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ revision: Optional[Union[str, Version]] = None,
+ token: Optional[Union[bool, str]] = None,
+ **config_kwargs,
+):
+ """Get the meta information about a dataset, returned as a dict mapping config name to DatasetInfoDict.
+
+ Args:
+ path (`str`): path to the dataset repository. Can be either:
+
+ - a local path to the dataset directory containing the data files,
+ e.g. `'./dataset/squad'`
+ - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`huggingface_hub.list_datasets`]),
+ e.g. `'rajpurkar/squad'`, `'nyu-mll/glue'` or``'openai/webtext'`
+ revision (`Union[str, datasets.Version]`, *optional*):
+ If specified, the dataset module will be loaded from the datasets repository at this version.
+ By default:
+ - it is set to the local version of the lib.
+ - it will also try to load it from the main branch if it's not available at the local version of the lib.
+ Specifying a version that is different from your local version of the lib might cause compatibility issues.
+ download_config ([`DownloadConfig`], *optional*):
+ Specific download configuration parameters.
+ download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`):
+ Download/generate mode.
+ data_files (`Union[Dict, List, str]`, *optional*):
+ Defining the data_files of the dataset configuration.
+ token (`str` or `bool`, *optional*):
+ Optional string or boolean to use as Bearer token for remote files on the Datasets Hub.
+ If `True`, or not specified, will get token from `"~/.huggingface"`.
+ **config_kwargs (additional keyword arguments):
+ Optional attributes for builder class which will override the attributes if supplied.
+
+ Example:
+
+ ```py
+ >>> from datasets import get_dataset_infos
+ >>> get_dataset_infos('cornell-movie-review-data/rotten_tomatoes')
+ {'default': DatasetInfo(description="Movie Review Dataset.\nThis is a dataset of containing 5,331 positive and 5,331 negative processed\nsentences from Rotten Tomatoes movie reviews...), ...}
+ ```
+ """
+ config_names = get_dataset_config_names(
+ path=path,
+ revision=revision,
+ download_config=download_config,
+ download_mode=download_mode,
+ data_files=data_files,
+ token=token,
+ )
+ return {
+ config_name: get_dataset_config_info(
+ path=path,
+ config_name=config_name,
+ data_files=data_files,
+ download_config=download_config,
+ download_mode=download_mode,
+ revision=revision,
+ token=token,
+ **config_kwargs,
+ )
+ for config_name in config_names
+ }
+
+
+def get_dataset_config_names(
+ path: str,
+ revision: Optional[Union[str, Version]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ data_files: Optional[Union[dict, list, str]] = None,
+ **download_kwargs,
+):
+ """Get the list of available config names for a particular dataset.
+
+ Args:
+ path (`str`): path to the dataset repository. Can be either:
+
+ - a local path to the dataset directory containing the data files,
+ e.g. `'./dataset/squad'`
+ - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`huggingface_hub.list_datasets`]),
+ e.g. `'rajpurkar/squad'`, `'nyu-mll/glue'` or``'openai/webtext'`
+ revision (`Union[str, datasets.Version]`, *optional*):
+ If specified, the dataset module will be loaded from the datasets repository at this version.
+ By default:
+ - it is set to the local version of the lib.
+ - it will also try to load it from the main branch if it's not available at the local version of the lib.
+ Specifying a version that is different from your local version of the lib might cause compatibility issues.
+ download_config ([`DownloadConfig`], *optional*):
+ Specific download configuration parameters.
+ download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`):
+ Download/generate mode.
+ data_files (`Union[Dict, List, str]`, *optional*):
+ Defining the data_files of the dataset configuration.
+ **download_kwargs (additional keyword arguments):
+ Optional attributes for [`DownloadConfig`] which will override the attributes in `download_config` if supplied,
+ for example `token`.
+
+ Example:
+
+ ```py
+ >>> from datasets import get_dataset_config_names
+ >>> get_dataset_config_names("nyu-mll/glue")
+ ['cola',
+ 'sst2',
+ 'mrpc',
+ 'qqp',
+ 'stsb',
+ 'mnli',
+ 'mnli_mismatched',
+ 'mnli_matched',
+ 'qnli',
+ 'rte',
+ 'wnli',
+ 'ax']
+ ```
+ """
+ dataset_module = dataset_module_factory(
+ path,
+ revision=revision,
+ download_config=download_config,
+ download_mode=download_mode,
+ data_files=data_files,
+ **download_kwargs,
+ )
+ builder_cls = get_dataset_builder_class(dataset_module, dataset_name=os.path.basename(path))
+ return list(builder_cls.builder_configs.keys()) or [
+ dataset_module.builder_kwargs.get("config_name", builder_cls.DEFAULT_CONFIG_NAME or "default")
+ ]
+
+
+def get_dataset_default_config_name(
+ path: str,
+ revision: Optional[Union[str, Version]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ data_files: Optional[Union[dict, list, str]] = None,
+ **download_kwargs,
+) -> Optional[str]:
+ """Get the default config name for a particular dataset.
+ Can return None only if the dataset has multiple configurations and no default configuration.
+
+ Args:
+ path (`str`): path to the dataset repository. Can be either:
+
+ - a local path to the dataset directory containing the data files,
+ e.g. `'./dataset/squad'`
+ - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`huggingface_hub.list_datasets`]),
+ e.g. `'rajpurkar/squad'`, `'nyu-mll/glue'` or``'openai/webtext'`
+ revision (`Union[str, datasets.Version]`, *optional*):
+ If specified, the dataset module will be loaded from the datasets repository at this version.
+ By default:
+ - it is set to the local version of the lib.
+ - it will also try to load it from the main branch if it's not available at the local version of the lib.
+ Specifying a version that is different from your local version of the lib might cause compatibility issues.
+ download_config ([`DownloadConfig`], *optional*):
+ Specific download configuration parameters.
+ download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`):
+ Download/generate mode.
+ data_files (`Union[Dict, List, str]`, *optional*):
+ Defining the data_files of the dataset configuration.
+ **download_kwargs (additional keyword arguments):
+ Optional attributes for [`DownloadConfig`] which will override the attributes in `download_config` if supplied,
+ for example `token`.
+
+ Returns:
+ Optional[str]: the default config name if there is one
+
+ Example:
+
+ ```py
+ >>> from datasets import get_dataset_default_config_name
+ >>> get_dataset_default_config_name("openbookqa")
+ 'main'
+ ```
+ """
+ dataset_module = dataset_module_factory(
+ path,
+ revision=revision,
+ download_config=download_config,
+ download_mode=download_mode,
+ data_files=data_files,
+ **download_kwargs,
+ )
+ builder_cls = get_dataset_builder_class(dataset_module, dataset_name=os.path.basename(path))
+ builder_configs = list(builder_cls.builder_configs.keys())
+ if builder_configs:
+ default_config_name = builder_configs[0] if len(builder_configs) == 1 else None
+ else:
+ default_config_name = "default"
+ return builder_cls.DEFAULT_CONFIG_NAME or default_config_name
+
+
+def get_dataset_config_info(
+ path: str,
+ config_name: Optional[str] = None,
+ data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ revision: Optional[Union[str, Version]] = None,
+ token: Optional[Union[bool, str]] = None,
+ **config_kwargs,
+) -> DatasetInfo:
+ """Get the meta information (DatasetInfo) about a dataset for a particular config
+
+ Args:
+ path (`str`): path to the dataset repository. Can be either:
+
+ - a local path to the dataset directory containing the data files,
+ e.g. `'./dataset/squad'`
+ - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`huggingface_hub.list_datasets`]),
+ e.g. `'rajpurkar/squad'`, `'nyu-mll/glue'` or``'openai/webtext'`
+ config_name (:obj:`str`, optional): Defining the name of the dataset configuration.
+ data_files (:obj:`str` or :obj:`Sequence` or :obj:`Mapping`, optional): Path(s) to source data file(s).
+ download_config (:class:`~download.DownloadConfig`, optional): Specific download configuration parameters.
+ download_mode (:class:`DownloadMode` or :obj:`str`, default ``REUSE_DATASET_IF_EXISTS``): Download/generate mode.
+ revision (:class:`~utils.Version` or :obj:`str`, optional): Version of the dataset to load.
+ As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch.
+ You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository.
+ token (``str`` or :obj:`bool`, optional): Optional string or boolean to use as Bearer token for remote files on the Datasets Hub.
+ If True, or not specified, will get token from `"~/.huggingface"`.
+ **config_kwargs (additional keyword arguments): optional attributes for builder class which will override the attributes if supplied.
+
+ """
+ builder = load_dataset_builder(
+ path,
+ name=config_name,
+ data_files=data_files,
+ download_config=download_config,
+ download_mode=download_mode,
+ revision=revision,
+ token=token,
+ **config_kwargs,
+ )
+ info = builder.info
+ if info.splits is None:
+ download_config = download_config.copy() if download_config else DownloadConfig()
+ if token is not None:
+ download_config.token = token
+ try:
+ info.splits = {
+ split_generator.name: {"name": split_generator.name, "dataset_name": path}
+ for split_generator in builder._split_generators(
+ StreamingDownloadManager(base_path=builder.base_path, download_config=download_config)
+ )
+ }
+ except Exception as err:
+ raise SplitsNotFoundError("The split names could not be parsed from the dataset config.") from err
+ return info
+
+
+def get_dataset_split_names(
+ path: str,
+ config_name: Optional[str] = None,
+ data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ revision: Optional[Union[str, Version]] = None,
+ token: Optional[Union[bool, str]] = None,
+ **config_kwargs,
+):
+ """Get the list of available splits for a particular config and dataset.
+
+ Args:
+ path (`str`): path to the dataset repository. Can be either:
+
+ - a local path to the dataset directory containing the data files,
+ e.g. `'./dataset/squad'`
+ - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`huggingface_hub.list_datasets`]),
+ e.g. `'rajpurkar/squad'`, `'nyu-mll/glue'` or``'openai/webtext'`
+ config_name (`str`, *optional*):
+ Defining the name of the dataset configuration.
+ data_files (`str` or `Sequence` or `Mapping`, *optional*):
+ Path(s) to source data file(s).
+ download_config ([`DownloadConfig`], *optional*):
+ Specific download configuration parameters.
+ download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`):
+ Download/generate mode.
+ revision ([`Version`] or `str`, *optional*):
+ Version of the dataset to load.
+ As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch.
+ You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository.
+ token (`str` or `bool`, *optional*):
+ Optional string or boolean to use as Bearer token for remote files on the Datasets Hub.
+ If `True`, or not specified, will get token from `"~/.huggingface"`.
+ **config_kwargs (additional keyword arguments):
+ Optional attributes for builder class which will override the attributes if supplied.
+
+ Example:
+
+ ```py
+ >>> from datasets import get_dataset_split_names
+ >>> get_dataset_split_names('cornell-movie-review-data/rotten_tomatoes')
+ ['train', 'validation', 'test']
+ ```
+ """
+ info = get_dataset_config_info(
+ path,
+ config_name=config_name,
+ data_files=data_files,
+ download_config=download_config,
+ download_mode=download_mode,
+ revision=revision,
+ token=token,
+ **config_kwargs,
+ )
+ return list(info.splits.keys())
diff --git a/venv/Lib/site-packages/datasets/io/__init__.py b/venv/Lib/site-packages/datasets/io/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/io/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/io/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3d3a4b7f1438e971cd9e03657ca895bdf1c96932
Binary files /dev/null and b/venv/Lib/site-packages/datasets/io/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/io/__pycache__/abc.cpython-311.pyc b/venv/Lib/site-packages/datasets/io/__pycache__/abc.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d322181d0f48cd673980ceb5cb5009960563c05b
Binary files /dev/null and b/venv/Lib/site-packages/datasets/io/__pycache__/abc.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/io/__pycache__/csv.cpython-311.pyc b/venv/Lib/site-packages/datasets/io/__pycache__/csv.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..93f3c601be39d5ed616915befdfccd1fc03879e0
Binary files /dev/null and b/venv/Lib/site-packages/datasets/io/__pycache__/csv.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/io/__pycache__/generator.cpython-311.pyc b/venv/Lib/site-packages/datasets/io/__pycache__/generator.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b622a954df3ff98e7b0dc67b82f11d8e7df5502c
Binary files /dev/null and b/venv/Lib/site-packages/datasets/io/__pycache__/generator.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/io/__pycache__/json.cpython-311.pyc b/venv/Lib/site-packages/datasets/io/__pycache__/json.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b410011cc68fb2b2eb2e26ae601ffcef010a8cbc
Binary files /dev/null and b/venv/Lib/site-packages/datasets/io/__pycache__/json.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/io/__pycache__/parquet.cpython-311.pyc b/venv/Lib/site-packages/datasets/io/__pycache__/parquet.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..18e4dc7c45a17730be85126e75c20cd6bb7ff857
Binary files /dev/null and b/venv/Lib/site-packages/datasets/io/__pycache__/parquet.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/io/__pycache__/spark.cpython-311.pyc b/venv/Lib/site-packages/datasets/io/__pycache__/spark.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c990c484af69672af50a2f4f14dfcb16b7e05b82
Binary files /dev/null and b/venv/Lib/site-packages/datasets/io/__pycache__/spark.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/io/__pycache__/sql.cpython-311.pyc b/venv/Lib/site-packages/datasets/io/__pycache__/sql.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f47d7728508fbea5b01e07ba24d36e07b3051c00
Binary files /dev/null and b/venv/Lib/site-packages/datasets/io/__pycache__/sql.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/io/__pycache__/text.cpython-311.pyc b/venv/Lib/site-packages/datasets/io/__pycache__/text.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f6d7033ad31ed7116afa8bf1eda7f71a37f78507
Binary files /dev/null and b/venv/Lib/site-packages/datasets/io/__pycache__/text.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/io/abc.py b/venv/Lib/site-packages/datasets/io/abc.py
new file mode 100644
index 0000000000000000000000000000000000000000..a1913cc20e3fd748ef912e2fb3d7c1e18f16ac8c
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/io/abc.py
@@ -0,0 +1,53 @@
+from abc import ABC, abstractmethod
+from typing import Optional, Union
+
+from .. import Dataset, DatasetDict, Features, IterableDataset, IterableDatasetDict, NamedSplit
+from ..utils.typing import NestedDataStructureLike, PathLike
+
+
+class AbstractDatasetReader(ABC):
+ def __init__(
+ self,
+ path_or_paths: Optional[NestedDataStructureLike[PathLike]] = None,
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ streaming: bool = False,
+ num_proc: Optional[int] = None,
+ **kwargs,
+ ):
+ self.path_or_paths = path_or_paths
+ self.split = split if split or isinstance(path_or_paths, dict) else "train"
+ self.features = features
+ self.cache_dir = cache_dir
+ self.keep_in_memory = keep_in_memory
+ self.streaming = streaming
+ self.num_proc = num_proc
+ self.kwargs = kwargs
+
+ @abstractmethod
+ def read(self) -> Union[Dataset, DatasetDict, IterableDataset, IterableDatasetDict]:
+ pass
+
+
+class AbstractDatasetInputStream(ABC):
+ def __init__(
+ self,
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ streaming: bool = False,
+ num_proc: Optional[int] = None,
+ **kwargs,
+ ):
+ self.features = features
+ self.cache_dir = cache_dir
+ self.keep_in_memory = keep_in_memory
+ self.streaming = streaming
+ self.num_proc = num_proc
+ self.kwargs = kwargs
+
+ @abstractmethod
+ def read(self) -> Union[Dataset, IterableDataset]:
+ pass
diff --git a/venv/Lib/site-packages/datasets/io/csv.py b/venv/Lib/site-packages/datasets/io/csv.py
new file mode 100644
index 0000000000000000000000000000000000000000..13d87d0401fafaa20c32908063e41bba8e349c24
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/io/csv.py
@@ -0,0 +1,143 @@
+import multiprocessing
+import os
+from typing import BinaryIO, Optional, Union
+
+import fsspec
+
+from .. import Dataset, Features, NamedSplit, config
+from ..formatting import query_table
+from ..packaged_modules.csv.csv import Csv
+from ..utils import tqdm as hf_tqdm
+from ..utils.typing import NestedDataStructureLike, PathLike
+from .abc import AbstractDatasetReader
+
+
+class CsvDatasetReader(AbstractDatasetReader):
+ def __init__(
+ self,
+ path_or_paths: NestedDataStructureLike[PathLike],
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ streaming: bool = False,
+ num_proc: Optional[int] = None,
+ **kwargs,
+ ):
+ super().__init__(
+ path_or_paths,
+ split=split,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ streaming=streaming,
+ num_proc=num_proc,
+ **kwargs,
+ )
+ path_or_paths = path_or_paths if isinstance(path_or_paths, dict) else {self.split: path_or_paths}
+ self.builder = Csv(
+ cache_dir=cache_dir,
+ data_files=path_or_paths,
+ features=features,
+ **kwargs,
+ )
+
+ def read(self):
+ # Build iterable dataset
+ if self.streaming:
+ dataset = self.builder.as_streaming_dataset(split=self.split)
+ # Build regular (map-style) dataset
+ else:
+ download_config = None
+ download_mode = None
+ verification_mode = None
+ base_path = None
+
+ self.builder.download_and_prepare(
+ download_config=download_config,
+ download_mode=download_mode,
+ verification_mode=verification_mode,
+ base_path=base_path,
+ num_proc=self.num_proc,
+ )
+ dataset = self.builder.as_dataset(split=self.split, in_memory=self.keep_in_memory)
+ return dataset
+
+
+class CsvDatasetWriter:
+ def __init__(
+ self,
+ dataset: Dataset,
+ path_or_buf: Union[PathLike, BinaryIO],
+ batch_size: Optional[int] = None,
+ num_proc: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ **to_csv_kwargs,
+ ):
+ if num_proc is not None and num_proc <= 0:
+ raise ValueError(f"num_proc {num_proc} must be an integer > 0.")
+
+ self.dataset = dataset
+ self.path_or_buf = path_or_buf
+ self.batch_size = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
+ self.num_proc = num_proc
+ self.encoding = "utf-8"
+ self.storage_options = storage_options or {}
+ self.to_csv_kwargs = to_csv_kwargs
+
+ def write(self) -> int:
+ _ = self.to_csv_kwargs.pop("path_or_buf", None)
+ header = self.to_csv_kwargs.pop("header", True)
+ index = self.to_csv_kwargs.pop("index", False)
+
+ if isinstance(self.path_or_buf, (str, bytes, os.PathLike)):
+ with fsspec.open(self.path_or_buf, "wb", **(self.storage_options or {})) as buffer:
+ written = self._write(file_obj=buffer, header=header, index=index, **self.to_csv_kwargs)
+ else:
+ written = self._write(file_obj=self.path_or_buf, header=header, index=index, **self.to_csv_kwargs)
+ return written
+
+ def _batch_csv(self, args):
+ offset, header, index, to_csv_kwargs = args
+
+ batch = query_table(
+ table=self.dataset.data,
+ key=slice(offset, offset + self.batch_size),
+ indices=self.dataset._indices,
+ )
+ csv_str = batch.to_pandas().to_csv(
+ path_or_buf=None, header=header if (offset == 0) else False, index=index, **to_csv_kwargs
+ )
+ return csv_str.encode(self.encoding)
+
+ def _write(self, file_obj: BinaryIO, header, index, **to_csv_kwargs) -> int:
+ """Writes the pyarrow table as CSV to a binary file handle.
+
+ Caller is responsible for opening and closing the handle.
+ """
+ written = 0
+
+ if self.num_proc is None or self.num_proc == 1:
+ for offset in hf_tqdm(
+ range(0, len(self.dataset), self.batch_size),
+ unit="ba",
+ desc="Creating CSV from Arrow format",
+ ):
+ csv_str = self._batch_csv((offset, header, index, to_csv_kwargs))
+ written += file_obj.write(csv_str)
+
+ else:
+ num_rows, batch_size = len(self.dataset), self.batch_size
+ with multiprocessing.Pool(self.num_proc) as pool:
+ for csv_str in hf_tqdm(
+ pool.imap(
+ self._batch_csv,
+ [(offset, header, index, to_csv_kwargs) for offset in range(0, num_rows, batch_size)],
+ ),
+ total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size,
+ unit="ba",
+ desc="Creating CSV from Arrow format",
+ ):
+ written += file_obj.write(csv_str)
+
+ return written
diff --git a/venv/Lib/site-packages/datasets/io/generator.py b/venv/Lib/site-packages/datasets/io/generator.py
new file mode 100644
index 0000000000000000000000000000000000000000..cb5ae767b8a13c66643c6ee3d7ec868b5924bc61
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/io/generator.py
@@ -0,0 +1,62 @@
+from typing import Callable, Optional
+
+from .. import Features, NamedSplit, Split
+from ..packaged_modules.generator.generator import Generator
+from .abc import AbstractDatasetInputStream
+
+
+class GeneratorDatasetInputStream(AbstractDatasetInputStream):
+ def __init__(
+ self,
+ generator: Callable,
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ streaming: bool = False,
+ gen_kwargs: Optional[dict] = None,
+ num_proc: Optional[int] = None,
+ split: NamedSplit = Split.TRAIN,
+ fingerprint: Optional[str] = None,
+ **kwargs,
+ ):
+ super().__init__(
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ streaming=streaming,
+ num_proc=num_proc,
+ **kwargs,
+ )
+ self.builder = Generator(
+ cache_dir=cache_dir,
+ features=features,
+ generator=generator,
+ gen_kwargs=gen_kwargs,
+ split=split,
+ config_id="default-fingerprint=" + fingerprint if fingerprint else None,
+ **kwargs,
+ )
+ self.fingerprint = fingerprint
+
+ def read(self):
+ # Build iterable dataset
+ if self.streaming:
+ dataset = self.builder.as_streaming_dataset(split=self.builder.config.split)
+ # Build regular (map-style) dataset
+ else:
+ download_config = None
+ download_mode = None
+ verification_mode = None
+ base_path = None
+
+ self.builder.download_and_prepare(
+ download_config=download_config,
+ download_mode=download_mode,
+ verification_mode=verification_mode,
+ base_path=base_path,
+ num_proc=self.num_proc,
+ )
+ dataset = self.builder.as_dataset(split=self.builder.config.split, in_memory=self.keep_in_memory)
+ if self.fingerprint:
+ dataset._fingerprint = self.fingerprint
+ return dataset
diff --git a/venv/Lib/site-packages/datasets/io/json.py b/venv/Lib/site-packages/datasets/io/json.py
new file mode 100644
index 0000000000000000000000000000000000000000..eb347aea7aa54fee182496a35d7fdf0fef06c787
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/io/json.py
@@ -0,0 +1,178 @@
+import multiprocessing
+import os
+from functools import partial
+from typing import BinaryIO, Optional, Union
+
+import fsspec
+
+from .. import Dataset, Features, NamedSplit, config
+from ..formatting import query_table
+from ..packaged_modules.json.json import Json
+from ..utils import tqdm as hf_tqdm
+from ..utils.json import get_json_field_paths_from_feature, json_decode_field
+from ..utils.typing import NestedDataStructureLike, PathLike
+from .abc import AbstractDatasetReader
+
+
+class JsonDatasetReader(AbstractDatasetReader):
+ def __init__(
+ self,
+ path_or_paths: NestedDataStructureLike[PathLike],
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ streaming: bool = False,
+ field: Optional[str] = None,
+ num_proc: Optional[int] = None,
+ **kwargs,
+ ):
+ super().__init__(
+ path_or_paths,
+ split=split,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ streaming=streaming,
+ num_proc=num_proc,
+ **kwargs,
+ )
+ self.field = field
+ path_or_paths = path_or_paths if isinstance(path_or_paths, dict) else {self.split: path_or_paths}
+ self.builder = Json(
+ cache_dir=cache_dir,
+ data_files=path_or_paths,
+ features=features,
+ field=field,
+ **kwargs,
+ )
+
+ def read(self):
+ # Build iterable dataset
+ if self.streaming:
+ dataset = self.builder.as_streaming_dataset(split=self.split)
+ # Build regular (map-style) dataset
+ else:
+ download_config = None
+ download_mode = None
+ verification_mode = None
+ base_path = None
+
+ self.builder.download_and_prepare(
+ download_config=download_config,
+ download_mode=download_mode,
+ verification_mode=verification_mode,
+ base_path=base_path,
+ num_proc=self.num_proc,
+ )
+ dataset = self.builder.as_dataset(split=self.split, in_memory=self.keep_in_memory)
+ return dataset
+
+
+class JsonDatasetWriter:
+ def __init__(
+ self,
+ dataset: Dataset,
+ path_or_buf: Union[PathLike, BinaryIO],
+ batch_size: Optional[int] = None,
+ num_proc: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ **to_json_kwargs,
+ ):
+ if num_proc is not None and num_proc <= 0:
+ raise ValueError(f"num_proc {num_proc} must be an integer > 0.")
+
+ self.dataset = dataset
+ self.path_or_buf = path_or_buf
+ self.batch_size = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
+ self.num_proc = num_proc
+ self.encoding = "utf-8"
+ self.storage_options = storage_options or {}
+ self.to_json_kwargs = to_json_kwargs
+
+ def write(self) -> int:
+ _ = self.to_json_kwargs.pop("path_or_buf", None)
+ orient = self.to_json_kwargs.pop("orient", "records")
+ lines = self.to_json_kwargs.pop("lines", True if orient == "records" else False)
+ if "index" not in self.to_json_kwargs and orient in ["split", "table"]:
+ self.to_json_kwargs["index"] = False
+
+ # Determine the default compression value based on self.path_or_buf type
+ default_compression = "infer" if isinstance(self.path_or_buf, (str, bytes, os.PathLike)) else None
+ compression = self.to_json_kwargs.pop("compression", default_compression)
+
+ if compression not in [None, "infer", "gzip", "bz2", "xz"]:
+ raise NotImplementedError(f"`datasets` currently does not support {compression} compression")
+
+ if not lines and self.batch_size < self.dataset.num_rows:
+ raise NotImplementedError(
+ "Output JSON will not be formatted correctly when lines = False and batch_size < number of rows in the dataset. Use pandas.DataFrame.to_json() instead."
+ )
+
+ if isinstance(self.path_or_buf, (str, bytes, os.PathLike)):
+ with fsspec.open(
+ self.path_or_buf, "wb", compression=compression, **(self.storage_options or {})
+ ) as buffer:
+ written = self._write(file_obj=buffer, orient=orient, lines=lines, **self.to_json_kwargs)
+ else:
+ if compression:
+ raise NotImplementedError(
+ f"The compression parameter is not supported when writing to a buffer, but compression={compression}"
+ " was passed. Please provide a local path instead."
+ )
+ written = self._write(file_obj=self.path_or_buf, orient=orient, lines=lines, **self.to_json_kwargs)
+ return written
+
+ def _batch_json(self, args):
+ offset, orient, lines, to_json_kwargs = args
+
+ batch = query_table(
+ table=self.dataset.data,
+ key=slice(offset, offset + self.batch_size),
+ indices=self.dataset._indices,
+ )
+ batch = batch.to_pandas()
+ for json_field_path in get_json_field_paths_from_feature(self.dataset.features):
+ col, *json_field_subpath = json_field_path
+ batch[col] = batch[col].apply(partial(json_decode_field, json_field_path=json_field_subpath))
+ json_str = batch.to_json(path_or_buf=None, orient=orient, lines=lines, **to_json_kwargs)
+ if not json_str.endswith("\n"):
+ json_str += "\n"
+ return json_str.encode(self.encoding)
+
+ def _write(
+ self,
+ file_obj: BinaryIO,
+ orient,
+ lines,
+ **to_json_kwargs,
+ ) -> int:
+ """Writes the pyarrow table as JSON lines to a binary file handle.
+
+ Caller is responsible for opening and closing the handle.
+ """
+ written = 0
+
+ if self.num_proc is None or self.num_proc == 1:
+ for offset in hf_tqdm(
+ range(0, len(self.dataset), self.batch_size),
+ unit="ba",
+ desc="Creating json from Arrow format",
+ ):
+ json_str = self._batch_json((offset, orient, lines, to_json_kwargs))
+ written += file_obj.write(json_str)
+ else:
+ num_rows, batch_size = len(self.dataset), self.batch_size
+ with multiprocessing.Pool(self.num_proc) as pool:
+ for json_str in hf_tqdm(
+ pool.imap(
+ self._batch_json,
+ [(offset, orient, lines, to_json_kwargs) for offset in range(0, num_rows, batch_size)],
+ ),
+ total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size,
+ unit="ba",
+ desc="Creating json from Arrow format",
+ ):
+ written += file_obj.write(json_str)
+
+ return written
diff --git a/venv/Lib/site-packages/datasets/io/parquet.py b/venv/Lib/site-packages/datasets/io/parquet.py
new file mode 100644
index 0000000000000000000000000000000000000000..7d17fbfdfc5147514089f5beb0761e3a5b4aa320
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/io/parquet.py
@@ -0,0 +1,159 @@
+import json
+import os
+from typing import BinaryIO, Optional, Union
+
+import fsspec
+import pyarrow.parquet as pq
+
+from .. import Dataset, Features, NamedSplit, config
+from ..arrow_writer import get_writer_batch_size_from_data_size, get_writer_batch_size_from_features
+from ..features.features import require_storage_embed
+from ..formatting import query_table
+from ..packaged_modules import _PACKAGED_DATASETS_MODULES
+from ..packaged_modules.parquet.parquet import Parquet
+from ..utils import tqdm as hf_tqdm
+from ..utils.typing import NestedDataStructureLike, PathLike
+from .abc import AbstractDatasetReader
+
+
+class ParquetDatasetReader(AbstractDatasetReader):
+ def __init__(
+ self,
+ path_or_paths: NestedDataStructureLike[PathLike],
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ streaming: bool = False,
+ num_proc: Optional[int] = None,
+ **kwargs,
+ ):
+ super().__init__(
+ path_or_paths,
+ split=split,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ streaming=streaming,
+ num_proc=num_proc,
+ **kwargs,
+ )
+ path_or_paths = path_or_paths if isinstance(path_or_paths, dict) else {self.split: path_or_paths}
+ hash = _PACKAGED_DATASETS_MODULES["parquet"][1]
+ self.builder = Parquet(
+ cache_dir=cache_dir,
+ data_files=path_or_paths,
+ features=features,
+ hash=hash,
+ **kwargs,
+ )
+
+ def read(self):
+ # Build iterable dataset
+ if self.streaming:
+ dataset = self.builder.as_streaming_dataset(split=self.split)
+ # Build regular (map-style) dataset
+ else:
+ download_config = None
+ download_mode = None
+ verification_mode = None
+ base_path = None
+
+ self.builder.download_and_prepare(
+ download_config=download_config,
+ download_mode=download_mode,
+ verification_mode=verification_mode,
+ base_path=base_path,
+ num_proc=self.num_proc,
+ )
+ dataset = self.builder.as_dataset(split=self.split, in_memory=self.keep_in_memory)
+ return dataset
+
+
+class ParquetDatasetWriter:
+ def __init__(
+ self,
+ dataset: Dataset,
+ path_or_buf: Union[PathLike, BinaryIO],
+ batch_size: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ use_content_defined_chunking: Union[bool, dict] = True,
+ write_page_index: bool = True,
+ **parquet_writer_kwargs,
+ ):
+ self.dataset = dataset
+ self.path_or_buf = path_or_buf
+ self.batch_size = (
+ batch_size
+ or get_writer_batch_size_from_features(dataset.features)
+ or get_writer_batch_size_from_data_size(len(dataset), dataset._estimate_nbytes())
+ )
+ self.storage_options = storage_options or {}
+ self.parquet_writer_kwargs = parquet_writer_kwargs
+ if use_content_defined_chunking is True:
+ use_content_defined_chunking = config.DEFAULT_CDC_OPTIONS
+ self.use_content_defined_chunking = use_content_defined_chunking
+ self.write_page_index = write_page_index
+
+ def write(self) -> int:
+ if isinstance(self.path_or_buf, (str, bytes, os.PathLike)):
+ with fsspec.open(self.path_or_buf, "wb", **(self.storage_options or {})) as buffer:
+ written = self._write(
+ file_obj=buffer,
+ batch_size=self.batch_size,
+ **self.parquet_writer_kwargs,
+ )
+ else:
+ written = self._write(
+ file_obj=self.path_or_buf,
+ batch_size=self.batch_size,
+ **self.parquet_writer_kwargs,
+ )
+ return written
+
+ def _write(self, file_obj: BinaryIO, batch_size: int, **parquet_writer_kwargs) -> int:
+ """Writes the pyarrow table as Parquet to a binary file handle.
+
+ Caller is responsible for opening and closing the handle.
+ """
+ written = 0
+ _ = parquet_writer_kwargs.pop("path_or_buf", None)
+ schema = self.dataset.features.arrow_schema
+
+ writer = pq.ParquetWriter(
+ file_obj,
+ schema=schema,
+ use_content_defined_chunking=self.use_content_defined_chunking,
+ write_page_index=self.write_page_index,
+ compression={
+ col: "none" if require_storage_embed(feature) else "snappy"
+ for col, feature in self.dataset.features.items()
+ },
+ use_dictionary=[
+ col for col, feature in self.dataset.features.items() if not require_storage_embed(feature)
+ ],
+ column_encoding={
+ col: "PLAIN" for col, feature in self.dataset.features.items() if require_storage_embed(feature)
+ },
+ **parquet_writer_kwargs,
+ )
+
+ for offset in hf_tqdm(
+ range(0, len(self.dataset), batch_size),
+ unit="ba",
+ desc="Creating parquet from Arrow format",
+ ):
+ batch = query_table(
+ table=self.dataset._data,
+ key=slice(offset, offset + batch_size),
+ indices=self.dataset._indices,
+ )
+ writer.write_table(batch)
+ written += batch.nbytes
+
+ # TODO(kszucs): we may want to persist multiple parameters
+ if self.use_content_defined_chunking is not False:
+ writer.add_key_value_metadata({"content_defined_chunking": json.dumps(self.use_content_defined_chunking)})
+
+ writer.close()
+ return written
diff --git a/venv/Lib/site-packages/datasets/io/spark.py b/venv/Lib/site-packages/datasets/io/spark.py
new file mode 100644
index 0000000000000000000000000000000000000000..7562ba1fb5f77ed8f82374e3021fcb3a93b1da8d
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/io/spark.py
@@ -0,0 +1,57 @@
+from typing import Optional
+
+import pyspark
+
+from .. import Features, NamedSplit
+from ..download import DownloadMode
+from ..packaged_modules.spark.spark import Spark
+from .abc import AbstractDatasetReader
+
+
+class SparkDatasetReader(AbstractDatasetReader):
+ """A dataset reader that reads from a Spark DataFrame.
+
+ When caching, cache materialization is parallelized over Spark; an NFS that is accessible to the driver must be
+ provided. Streaming is not currently supported.
+ """
+
+ def __init__(
+ self,
+ df: pyspark.sql.DataFrame,
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ streaming: bool = True,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ working_dir: str = None,
+ load_from_cache_file: bool = True,
+ file_format: str = "arrow",
+ **kwargs,
+ ):
+ super().__init__(
+ split=split,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ streaming=streaming,
+ **kwargs,
+ )
+ self._load_from_cache_file = load_from_cache_file
+ self._file_format = file_format
+ self.builder = Spark(
+ df=df,
+ features=features,
+ cache_dir=cache_dir,
+ working_dir=working_dir,
+ **kwargs,
+ )
+
+ def read(self):
+ if self.streaming:
+ return self.builder.as_streaming_dataset(split=self.split)
+ download_mode = None if self._load_from_cache_file else DownloadMode.FORCE_REDOWNLOAD
+ self.builder.download_and_prepare(
+ download_mode=download_mode,
+ file_format=self._file_format,
+ )
+ return self.builder.as_dataset(split=self.split)
diff --git a/venv/Lib/site-packages/datasets/io/sql.py b/venv/Lib/site-packages/datasets/io/sql.py
new file mode 100644
index 0000000000000000000000000000000000000000..f74374a5ad07f9a9767976752fd34fdc8ad71540
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/io/sql.py
@@ -0,0 +1,122 @@
+import multiprocessing
+from typing import TYPE_CHECKING, Optional, Union
+
+from .. import Dataset, Features, config
+from ..formatting import query_table
+from ..packaged_modules.sql.sql import Sql
+from ..utils import tqdm as hf_tqdm
+from .abc import AbstractDatasetInputStream
+
+
+if TYPE_CHECKING:
+ import sqlite3
+
+ import sqlalchemy
+
+
+class SqlDatasetReader(AbstractDatasetInputStream):
+ def __init__(
+ self,
+ sql: Union[str, "sqlalchemy.sql.Selectable"],
+ con: Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"],
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ **kwargs,
+ ):
+ super().__init__(features=features, cache_dir=cache_dir, keep_in_memory=keep_in_memory, **kwargs)
+ self.builder = Sql(
+ cache_dir=cache_dir,
+ features=features,
+ sql=sql,
+ con=con,
+ **kwargs,
+ )
+
+ def read(self):
+ download_config = None
+ download_mode = None
+ verification_mode = None
+ base_path = None
+
+ self.builder.download_and_prepare(
+ download_config=download_config,
+ download_mode=download_mode,
+ verification_mode=verification_mode,
+ base_path=base_path,
+ )
+
+ # Build dataset for splits
+ dataset = self.builder.as_dataset(split="train", in_memory=self.keep_in_memory)
+ return dataset
+
+
+class SqlDatasetWriter:
+ def __init__(
+ self,
+ dataset: Dataset,
+ name: str,
+ con: Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"],
+ batch_size: Optional[int] = None,
+ num_proc: Optional[int] = None,
+ **to_sql_kwargs,
+ ):
+ if num_proc is not None and num_proc <= 0:
+ raise ValueError(f"num_proc {num_proc} must be an integer > 0.")
+
+ self.dataset = dataset
+ self.name = name
+ self.con = con
+ self.batch_size = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
+ self.num_proc = num_proc
+ self.to_sql_kwargs = to_sql_kwargs
+
+ def write(self) -> int:
+ _ = self.to_sql_kwargs.pop("sql", None)
+ _ = self.to_sql_kwargs.pop("con", None)
+ index = self.to_sql_kwargs.pop("index", False)
+
+ written = self._write(index=index, **self.to_sql_kwargs)
+ return written
+
+ def _batch_sql(self, args):
+ offset, index, to_sql_kwargs = args
+ to_sql_kwargs = {**to_sql_kwargs, "if_exists": "append"} if offset > 0 else to_sql_kwargs
+ batch = query_table(
+ table=self.dataset.data,
+ key=slice(offset, offset + self.batch_size),
+ indices=self.dataset._indices,
+ )
+ df = batch.to_pandas()
+ num_rows = df.to_sql(self.name, self.con, index=index, **to_sql_kwargs)
+ return num_rows or len(df)
+
+ def _write(self, index, **to_sql_kwargs) -> int:
+ """Writes the pyarrow table as SQL to a database.
+
+ Caller is responsible for opening and closing the SQL connection.
+ """
+ written = 0
+
+ if self.num_proc is None or self.num_proc == 1:
+ for offset in hf_tqdm(
+ range(0, len(self.dataset), self.batch_size),
+ unit="ba",
+ desc="Creating SQL from Arrow format",
+ ):
+ written += self._batch_sql((offset, index, to_sql_kwargs))
+ else:
+ num_rows, batch_size = len(self.dataset), self.batch_size
+ with multiprocessing.Pool(self.num_proc) as pool:
+ for num_rows in hf_tqdm(
+ pool.imap(
+ self._batch_sql,
+ [(offset, index, to_sql_kwargs) for offset in range(0, num_rows, batch_size)],
+ ),
+ total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size,
+ unit="ba",
+ desc="Creating SQL from Arrow format",
+ ):
+ written += num_rows
+
+ return written
diff --git a/venv/Lib/site-packages/datasets/io/text.py b/venv/Lib/site-packages/datasets/io/text.py
new file mode 100644
index 0000000000000000000000000000000000000000..d010ddcbd62515e99165a86f36d483b2dc438b3f
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/io/text.py
@@ -0,0 +1,58 @@
+from typing import Optional
+
+from .. import Features, NamedSplit
+from ..packaged_modules.text.text import Text
+from ..utils.typing import NestedDataStructureLike, PathLike
+from .abc import AbstractDatasetReader
+
+
+class TextDatasetReader(AbstractDatasetReader):
+ def __init__(
+ self,
+ path_or_paths: NestedDataStructureLike[PathLike],
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ cache_dir: str = None,
+ keep_in_memory: bool = False,
+ streaming: bool = False,
+ num_proc: Optional[int] = None,
+ **kwargs,
+ ):
+ super().__init__(
+ path_or_paths,
+ split=split,
+ features=features,
+ cache_dir=cache_dir,
+ keep_in_memory=keep_in_memory,
+ streaming=streaming,
+ num_proc=num_proc,
+ **kwargs,
+ )
+ path_or_paths = path_or_paths if isinstance(path_or_paths, dict) else {self.split: path_or_paths}
+ self.builder = Text(
+ cache_dir=cache_dir,
+ data_files=path_or_paths,
+ features=features,
+ **kwargs,
+ )
+
+ def read(self):
+ # Build iterable dataset
+ if self.streaming:
+ dataset = self.builder.as_streaming_dataset(split=self.split)
+ # Build regular (map-style) dataset
+ else:
+ download_config = None
+ download_mode = None
+ verification_mode = None
+ base_path = None
+
+ self.builder.download_and_prepare(
+ download_config=download_config,
+ download_mode=download_mode,
+ verification_mode=verification_mode,
+ base_path=base_path,
+ num_proc=self.num_proc,
+ )
+ dataset = self.builder.as_dataset(split=self.split, in_memory=self.keep_in_memory)
+ return dataset
diff --git a/venv/Lib/site-packages/datasets/iterable_dataset.py b/venv/Lib/site-packages/datasets/iterable_dataset.py
new file mode 100644
index 0000000000000000000000000000000000000000..35cc7cc9092550944ae85cda4c0a5e3edfe620d1
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/iterable_dataset.py
@@ -0,0 +1,5393 @@
+import asyncio
+import concurrent.futures
+import contextlib
+import inspect
+import itertools
+import multiprocessing.pool
+import re
+import sys
+import tempfile
+import time
+from collections import Counter
+from collections.abc import Iterable, Iterator
+from copy import copy, deepcopy
+from dataclasses import dataclass
+from functools import partial
+from itertools import cycle, islice
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, BinaryIO, Callable, Optional, Union
+
+import fsspec.asyn
+import multiprocess as mp
+import numpy as np
+import pandas as pd
+import pyarrow as pa
+import pyarrow.dataset as pds
+import pyarrow.parquet as pq
+from huggingface_hub import (
+ CommitInfo,
+ CommitOperationAdd,
+ HfApi,
+ HfFileSystem,
+ HfFileSystemResolvedPath,
+)
+from huggingface_hub.utils import RepositoryNotFoundError
+from packaging import version
+
+from . import __version__, config
+from .arrow_dataset import Dataset, DatasetInfoMixin, _push_to_bucket, _push_to_repo
+from .features import Features
+from .features.features import (
+ FeatureType,
+ List,
+ Value,
+ _align_features,
+ _check_if_features_can_be_aligned,
+ _fix_for_backward_compatible_features,
+ _visit,
+ cast_to_python_objects,
+ require_decoding,
+)
+from .formatting import (
+ ArrowFormatter,
+ PythonFormatter,
+ TableFormatter,
+ TensorFormatter,
+ get_format_type_from_alias,
+ get_formatter,
+)
+from .info import DatasetInfo
+from .naming import _split_re
+from .splits import NamedSplit, Split, SplitInfo
+from .table import (
+ _batch_accumulate_arrow_table_by_columns,
+ _batch_arrow_table,
+ cast_table_to_features,
+ embed_table_storage,
+ read_schema_from_file,
+ table_cast,
+)
+from .utils import tqdm as hf_tqdm
+from .utils.logging import get_logger
+from .utils.py_utils import (
+ Literal,
+ convert_file_size_to_int,
+ iflatmap_unordered,
+)
+from .utils.sharding import _merge_gen_kwargs, _number_of_shards_in_gen_kwargs, _shuffle_gen_kwargs, _split_gen_kwargs
+from .utils.typing import PathLike
+
+
+if config.HF_HUB_VERSION >= version.parse("1.6.0"):
+ from huggingface_hub.errors import BucketNotFoundError
+ from huggingface_hub.hf_file_system import HfFileSystemResolvedBucketPath, HfFileSystemResolvedRepositoryPath
+
+else:
+ BucketNotFoundError = None
+ HfFileSystemResolvedBucketPath = None
+ HfFileSystemResolvedRepositoryPath = HfFileSystemResolvedPath
+
+if TYPE_CHECKING:
+ import sqlite3
+
+ import polars as pl
+ import sqlalchemy
+ import torch
+
+ from .builder import Key as BuilderKey
+
+logger = get_logger(__name__)
+
+Key = Union[int, str, tuple[int, int], "BuilderKey"]
+
+
+def identity_func(x):
+ return x
+
+
+def _rename_columns_fn(example: dict, column_mapping: dict[str, str]):
+ if any(col not in example for col in column_mapping):
+ raise ValueError(
+ f"Error when renaming {list(column_mapping)} to {list(column_mapping.values())}: columns {set(column_mapping) - set(example)} are not in the dataset."
+ )
+ if any(col in example for col in column_mapping.values()):
+ raise ValueError(
+ f"Error when renaming {list(column_mapping)} to {list(column_mapping.values())}: columns {set(example) - set(column_mapping.values())} are already in the dataset."
+ )
+ return {
+ new_column_name: example[original_column_name]
+ for original_column_name, new_column_name in column_mapping.items()
+ }
+
+
+def add_column_fn(example: dict, idx: int, name: str, column: list[dict]):
+ if name in example:
+ raise ValueError(f"Error when adding {name}: column {name} is already in the dataset.")
+ return {name: column[idx]}
+
+
+def _infer_features_from_batch(batch: dict[str, list], try_features: Optional[Features] = None) -> Features:
+ pa_table = pa.Table.from_pydict(batch)
+ if try_features is not None:
+ try:
+ pa_table = table_cast(pa_table, pa.schema(try_features.type))
+ except (TypeError, pa.ArrowInvalid, pa.ArrowNotImplementedError):
+ pass
+ return Features.from_arrow_schema(pa_table.schema)
+
+
+def _examples_to_batch(examples: list[dict[str, Any]]) -> dict[str, list]:
+ # we order the columns by order of appearance
+ # to do so, we use a dict as an ordered set
+ cols = {col: None for example in examples for col in example}
+ # when an example is missing a column, we set the value to None with .get()
+ arrays = [[example.get(col) for example in examples] for col in cols]
+ return dict(zip(cols, arrays))
+
+
+def _batch_to_examples(batch: dict[str, list]) -> Iterator[dict[str, Any]]:
+ """Convert a batch (dict of examples) to examples list"""
+ n_examples = 0 if len(batch) == 0 else len(batch[next(iter(batch))])
+ for i in range(n_examples):
+ yield {col: array[i] for col, array in batch.items()}
+
+
+def _convert_to_arrow(
+ iterable: Iterable[tuple[Key, dict]],
+ batch_size: int,
+ drop_last_batch: bool = False,
+) -> Iterator[tuple[Key, pa.Table]]:
+ """Convert and group examples in Arrow tables of size `batch_size`.
+
+ Args:
+ iterable (`Iterable[Tuple[Key, dict]]`):
+ An examples iterable containing tuples (example_key, example) of type (int/str, dict)
+ batch_size (`Optional[int]`):
+ Size of each sub-table to yield. If None or <= 0, yields the full table.
+ drop_last_batch (`bool`, defaults to `False`):
+ Drop the last batch if it is smaller than `batch_size`.
+ """
+ if batch_size is None or batch_size <= 0:
+ yield (
+ "all",
+ pa.Table.from_pylist(cast_to_python_objects([example for _, example in iterable], only_1d_for_numpy=True)),
+ )
+ return
+ iterator = iter(iterable)
+ for key, example in iterator:
+ iterator_batch = islice(iterator, batch_size - 1)
+ key_examples_list = [(key, example)] + list(iterator_batch)
+ if len(key_examples_list) < batch_size and drop_last_batch:
+ return
+ keys, examples = zip(*key_examples_list)
+ new_key = "_".join(str(key) for key in keys)
+ yield new_key, pa.Table.from_pylist(cast_to_python_objects(examples, only_1d_for_numpy=True))
+
+
+def shift_ex_examples_rngs(ex_iterable: "_BaseExamplesIterable", value: int) -> "_BaseExamplesIterable":
+ """We need to go through the ex_iterables recursively, create a new seed and return a new iterable, then set it to the containing ex_iterable."""
+
+ def set_seed_recursively(ex_iterable):
+ if hasattr(ex_iterable, "shift_rngs"):
+ ex_iterable = ex_iterable.shift_rngs(value)
+ if hasattr(ex_iterable, "ex_iterable"):
+ ex_iterable.ex_iterable = set_seed_recursively(ex_iterable.ex_iterable)
+ if hasattr(ex_iterable, "ex_iterables"):
+ ex_iterable.ex_iterables = [set_seed_recursively(ei) for ei in ex_iterable.ex_iterables]
+ return ex_iterable
+
+ return set_seed_recursively(ex_iterable)
+
+
+class _BaseExamplesIterable:
+ """Base class for the examples iterable used by an IterableDataset"""
+
+ def __init__(self) -> None:
+ self._state_dict: Optional[Union[list, dict]] = None
+
+ def __iter__(self) -> Iterator[tuple[Key, dict]]:
+ """An examples iterable should yield tuples (example_key, example) of type (int/str, dict)"""
+ raise NotImplementedError(f"{type(self)} doesn't implement __iter__ yet")
+
+ @property
+ def iter_arrow(self) -> Optional[Callable[[], Iterator[tuple[Key, pa.Table]]]]:
+ return None
+
+ @property
+ def is_typed(self) -> bool:
+ return False
+
+ @property
+ def features(self) -> Optional[Features]:
+ return None
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "_BaseExamplesIterable":
+ """
+ Either shuffle the shards/sources of the dataset, or propagate the shuffling to the underlying iterable.
+ If the order of the shards must stay fixed (when using .skip or .take for example), then this method returns self.
+ """
+ raise NotImplementedError(f"{type(self)} doesn't implement shuffle_data_sources yet")
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "_BaseExamplesIterable":
+ """Either keep only the requested shard, or propagate the request to the underlying iterable."""
+ raise NotImplementedError(f"{type(self)} doesn't implement shard_data_sources yet")
+
+ def reshard_data_sources(self) -> "_BaseExamplesIterable":
+ """
+ Either reshard the shards/sources of the dataset, i.e. further split the current shards into more shards,
+ or propagate the resharding to the underlying iterable.
+ If the examples iterable can't be further resharded, then this method returns self.
+ """
+ raise NotImplementedError(f"{type(self)} doesn't implement reshard_data_sources yet")
+
+ def split_shard_indices_by_worker(self, num_shards: int, index: int, contiguous=True) -> list[int]:
+ if contiguous:
+ div = self.num_shards // num_shards
+ mod = self.num_shards % num_shards
+ start = div * index + min(index, mod)
+ end = start + div + (1 if index < mod else 0)
+ return list(range(start, end))
+ else:
+ return list(range(index, self.num_shards, num_shards))
+
+ @property
+ def num_shards(self) -> int:
+ raise NotImplementedError(f"{type(self)} doesn't implement num_shards yet")
+
+ def _init_state_dict(self) -> dict:
+ raise NotImplementedError(f"{type(self)} doesn't implement _init_state_dict yet")
+
+ def load_state_dict(self, state_dict: dict) -> dict:
+ def _inner_load_state_dict(state, new_state):
+ if new_state is not None and isinstance(state, dict):
+ for key in new_state:
+ state[key] = _inner_load_state_dict(state[key], new_state[key])
+ return state
+ elif new_state is not None and isinstance(state, list):
+ for i in range(len(state)):
+ state[i] = _inner_load_state_dict(state[i], new_state[i])
+ return state
+ return deepcopy(new_state)
+
+ self._init_state_dict()
+ return _inner_load_state_dict(self._state_dict, state_dict)
+
+ def state_dict(self) -> dict:
+ if self._state_dict:
+ return deepcopy(self._state_dict)
+ raise RuntimeError("State dict is not initialized, please call ex_iterable._init_state_dict() first.")
+
+ @property
+ def sleep_on_threads_shutdown(self):
+ if hasattr(self, "_sleep_on_threads_shutdown"):
+ return self._sleep_on_threads_shutdown
+ else:
+ ex_iterables = [self.ex_iterable] if hasattr(self, "ex_iterable") else self.ex_iterables
+ return any(ex_iterable.sleep_on_threads_shutdown for ex_iterable in ex_iterables)
+
+
+class ExamplesIterable(_BaseExamplesIterable):
+ def __init__(
+ self,
+ generate_examples_fn: Callable[..., Iterator[tuple[Key, dict]]],
+ kwargs: dict,
+ generate_more_kwargs_fn: Optional[Callable[..., Iterator[dict]]] = None,
+ sleep_on_threads_shutdown: bool = False,
+ ):
+ super().__init__()
+ self.generate_examples_fn = generate_examples_fn
+ self.kwargs = kwargs
+
+ # for resharding
+ self.generate_more_kwargs_fn = generate_more_kwargs_fn
+
+ # for threads shutdowns
+ self._sleep_on_threads_shutdown = sleep_on_threads_shutdown
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = {"shard_idx": 0, "shard_example_idx": 0, "type": self.__class__.__name__}
+ return self._state_dict
+
+ def __iter__(self):
+ shard_idx_start = self._state_dict["shard_idx"] if self._state_dict else 0
+ for gen_kwargs in islice(_split_gen_kwargs(self.kwargs, max_num_jobs=self.num_shards), shard_idx_start, None):
+ shard_example_idx_start = self._state_dict["shard_example_idx"] if self._state_dict else 0
+ for key_example in islice(self.generate_examples_fn(**gen_kwargs), shard_example_idx_start, None):
+ if self._state_dict:
+ self._state_dict["shard_example_idx"] += 1
+ yield key_example
+ if self._state_dict:
+ self._state_dict["shard_idx"] += 1
+ self._state_dict["shard_example_idx"] = 0
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "ExamplesIterable":
+ return ExamplesIterable(
+ self.generate_examples_fn,
+ _shuffle_gen_kwargs(deepcopy(generator), self.kwargs),
+ self.generate_more_kwargs_fn,
+ self.sleep_on_threads_shutdown,
+ )
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "ExamplesIterable":
+ """Keep only the requested shard."""
+ gen_kwargs_list = _split_gen_kwargs(self.kwargs, max_num_jobs=self.num_shards)
+ shard_indices = self.split_shard_indices_by_worker(num_shards, index, contiguous=contiguous)
+ requested_gen_kwargs = _merge_gen_kwargs([gen_kwargs_list[i] for i in shard_indices])
+ return ExamplesIterable(
+ self.generate_examples_fn,
+ requested_gen_kwargs,
+ self.generate_more_kwargs_fn,
+ self.sleep_on_threads_shutdown,
+ )
+
+ def reshard_data_sources(self) -> "ExamplesIterable":
+ """Split shars into more shards if possible."""
+ if not self.generate_more_kwargs_fn:
+ return ExamplesIterable(
+ self.generate_examples_fn, self.kwargs, self.generate_more_kwargs_fn, self.sleep_on_threads_shutdown
+ )
+ gen_kwargs_list = _split_gen_kwargs(self.kwargs, max_num_jobs=self.num_shards)
+ new_gen_kwargs = _merge_gen_kwargs(
+ [
+ new_gen_kwargs
+ for gen_kwargs in gen_kwargs_list
+ for new_gen_kwargs in self.generate_more_kwargs_fn(**gen_kwargs)
+ ]
+ )
+ return ExamplesIterable(
+ self.generate_examples_fn, new_gen_kwargs, self.generate_more_kwargs_fn, self.sleep_on_threads_shutdown
+ )
+
+ @property
+ def num_shards(self) -> int:
+ return _number_of_shards_in_gen_kwargs(self.kwargs)
+
+
+class ArrowExamplesIterable(_BaseExamplesIterable):
+ def __init__(
+ self,
+ generate_tables_fn: Callable[..., Iterator[tuple[Key, pa.Table]]],
+ kwargs: dict,
+ generate_more_kwargs_fn: Optional[Callable[..., Iterator[dict]]] = None,
+ sleep_on_threads_shutdown: bool = False,
+ ):
+ super().__init__()
+ self.generate_tables_fn = generate_tables_fn
+ self.kwargs = kwargs
+
+ # for resharding
+ self.generate_more_kwargs_fn = generate_more_kwargs_fn
+
+ # for threads shutdowns
+ self._sleep_on_threads_shutdown = sleep_on_threads_shutdown
+
+ @property
+ def iter_arrow(self):
+ return self._iter_arrow
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = {"shard_idx": 0, "shard_example_idx": 0, "type": self.__class__.__name__}
+ return self._state_dict
+
+ def __iter__(self):
+ formatter = PythonFormatter()
+ shard_idx_start = self._state_dict["shard_idx"] if self._state_dict else 0
+ for gen_kwags in islice(_split_gen_kwargs(self.kwargs, max_num_jobs=self.num_shards), shard_idx_start, None):
+ shard_example_idx_start = self._state_dict["shard_example_idx"] if self._state_dict else 0
+ shard_example_idx = 0
+ for key, pa_table in self.generate_tables_fn(**gen_kwags):
+ if shard_example_idx + len(pa_table) <= shard_example_idx_start:
+ shard_example_idx += len(pa_table)
+ continue
+ for pa_subtable in pa_table.to_reader(max_chunksize=config.ARROW_READER_BATCH_SIZE_IN_DATASET_ITER):
+ formatted_batch = formatter.format_batch(pa_subtable)
+ for example in _batch_to_examples(formatted_batch):
+ if shard_example_idx >= shard_example_idx_start:
+ if self._state_dict:
+ self._state_dict["shard_example_idx"] += 1
+ yield key, example
+ shard_example_idx += 1
+ if self._state_dict:
+ self._state_dict["shard_idx"] += 1
+ self._state_dict["shard_example_idx"] = 0
+
+ def _iter_arrow(self):
+ shard_idx_start = self._state_dict["shard_idx"] if self._state_dict else 0
+ for gen_kwags in islice(_split_gen_kwargs(self.kwargs, max_num_jobs=self.num_shards), shard_idx_start, None):
+ shard_example_idx_start = self._state_dict["shard_example_idx"] if self._state_dict else 0
+ shard_example_idx = 0
+ for key, pa_table in self.generate_tables_fn(**gen_kwags):
+ shard_example_idx += len(pa_table)
+ if shard_example_idx <= shard_example_idx_start:
+ continue
+ if self._state_dict:
+ self._state_dict["shard_example_idx"] += len(pa_table)
+ yield key, pa_table
+ if self._state_dict:
+ self._state_dict["shard_idx"] += 1
+ self._state_dict["shard_example_idx"] = 0
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "ArrowExamplesIterable":
+ return ArrowExamplesIterable(
+ self.generate_tables_fn,
+ _shuffle_gen_kwargs(deepcopy(generator), self.kwargs),
+ self.generate_more_kwargs_fn,
+ self.sleep_on_threads_shutdown,
+ )
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "ArrowExamplesIterable":
+ """Keep only the requested shard."""
+ gen_kwargs_list = _split_gen_kwargs(self.kwargs, max_num_jobs=self.num_shards)
+ shard_indices = self.split_shard_indices_by_worker(num_shards, index, contiguous=contiguous)
+ requested_gen_kwargs = _merge_gen_kwargs([gen_kwargs_list[i] for i in shard_indices])
+ return ArrowExamplesIterable(
+ self.generate_tables_fn, requested_gen_kwargs, self.generate_more_kwargs_fn, self.sleep_on_threads_shutdown
+ )
+
+ def reshard_data_sources(self) -> "ArrowExamplesIterable":
+ """Split shars into more shards if possible."""
+ if not self.generate_more_kwargs_fn:
+ return ArrowExamplesIterable(self.generate_tables_fn, self.kwargs, self.generate_more_kwargs_fn)
+ gen_kwargs_list = _split_gen_kwargs(self.kwargs, max_num_jobs=self.num_shards)
+ new_gen_kwargs = _merge_gen_kwargs(
+ [
+ new_gen_kwargs
+ for gen_kwargs in gen_kwargs_list
+ for new_gen_kwargs in self.generate_more_kwargs_fn(**gen_kwargs)
+ ]
+ )
+ return ArrowExamplesIterable(
+ self.generate_tables_fn, new_gen_kwargs, self.generate_more_kwargs_fn, self.sleep_on_threads_shutdown
+ )
+
+ @property
+ def num_shards(self) -> int:
+ return _number_of_shards_in_gen_kwargs(self.kwargs)
+
+
+class RebatchedArrowExamplesIterable(_BaseExamplesIterable):
+ def __init__(
+ self,
+ ex_iterable: _BaseExamplesIterable,
+ batch_size: Optional[int],
+ drop_last_batch: bool = False,
+ force_convert_to_arrow: bool = False,
+ ):
+ super().__init__()
+ self.ex_iterable = ex_iterable
+ self.batch_size = batch_size
+ self.drop_last_batch = drop_last_batch
+ self.force_convert_to_arrow = force_convert_to_arrow
+
+ @property
+ def iter_arrow(self):
+ return self._iter_arrow if self.ex_iterable.iter_arrow or self.force_convert_to_arrow else None
+
+ @property
+ def is_typed(self):
+ return self.ex_iterable.is_typed
+
+ @property
+ def features(self):
+ return self.ex_iterable.features
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = {
+ "examples_iterable": self.ex_iterable._init_state_dict(),
+ "previous_state": None,
+ "batch_idx": 0,
+ "num_chunks_since_previous_state": 0,
+ "cropped_chunk_length": 0,
+ "type": self.__class__.__name__,
+ }
+ return self._state_dict
+
+ def __iter__(self):
+ yield from self.ex_iterable
+
+ def _iter_arrow(self) -> Iterator[tuple[Key, pa.Table]]:
+ """Iterate over sub-tables of size `batch_size`."""
+ if self._state_dict and self._state_dict["previous_state"]:
+ self.ex_iterable.load_state_dict(self._state_dict["previous_state"])
+ if self.ex_iterable.iter_arrow:
+ iterator = self.ex_iterable.iter_arrow()
+ elif self.force_convert_to_arrow:
+ iterator = _convert_to_arrow(self.ex_iterable, batch_size=1)
+ else:
+ raise RuntimeError(
+ "_iter_arrow is not available in RebatchedArrowExamplesIterable, use an examples iterable that implements _iter_arrow() or pass force_convert_to_arrow=True"
+ )
+ if self.batch_size is None or self.batch_size <= 0:
+ if self._state_dict and self._state_dict["batch_idx"] > 0:
+ return
+ all_pa_table = pa.concat_tables([pa_table for _, pa_table in iterator])
+ if self._state_dict:
+ self._state_dict["batch_idx"] = 1
+ yield "all", all_pa_table
+ return
+ keys_buffer = []
+ chunks_buffer = []
+ chunks_buffer_size = 0
+ num_chunks_to_skip = self._state_dict["num_chunks_since_previous_state"] if self._state_dict else 0
+ chunk_length_to_crop = self._state_dict["cropped_chunk_length"] if self._state_dict else 0
+ if self._state_dict:
+ previous_state = self.ex_iterable.state_dict()
+ self._state_dict["previous_state"] = previous_state
+ for key, pa_table in iterator:
+ for num_chunks_since_previous_state, chunk in enumerate(pa_table.to_reader(max_chunksize=self.batch_size)):
+ if num_chunks_to_skip > 1:
+ num_chunks_to_skip -= 1
+ continue
+ elif num_chunks_to_skip == 1 and chunk_length_to_crop == 0:
+ num_chunks_to_skip -= 1
+ continue
+ elif num_chunks_to_skip == 1 and chunk_length_to_crop > 0:
+ chunk = chunk.slice(chunk_length_to_crop, len(chunk) - chunk_length_to_crop)
+ num_chunks_to_skip = 0
+ chunk_length_to_crop = 0
+ if len(chunk) == 0:
+ continue
+
+ if chunks_buffer_size + len(chunk) < self.batch_size:
+ keys_buffer.append(key)
+ chunks_buffer.append(chunk)
+ chunks_buffer_size += len(chunk)
+ continue
+ elif chunks_buffer_size + len(chunk) == self.batch_size:
+ keys_buffer.append(key)
+ chunks_buffer.append(chunk)
+ new_key = "_".join(str(_key) for _key in keys_buffer)
+ if self._state_dict:
+ self._state_dict["batch_idx"] += 1
+ self._state_dict["num_chunks_since_previous_state"] += len(chunks_buffer)
+ self._state_dict["cropped_chunk_length"] = 0
+ yield new_key, pa.Table.from_batches(chunks_buffer)
+ keys_buffer = []
+ chunks_buffer = []
+ chunks_buffer_size = 0
+ if self._state_dict:
+ self._state_dict["previous_state"] = previous_state
+ self._state_dict["num_chunks_since_previous_state"] = num_chunks_since_previous_state + 1
+ else:
+ cropped_chunk_length = self.batch_size - chunks_buffer_size
+ keys_buffer.append(f"{key}[:{cropped_chunk_length}]")
+ chunks_buffer.append(chunk.slice(0, cropped_chunk_length))
+ new_key = "_".join(str(_key) for _key in keys_buffer)
+ if self._state_dict:
+ self._state_dict["batch_idx"] += 1
+ self._state_dict["num_chunks_since_previous_state"] += len(chunks_buffer)
+ self._state_dict["cropped_chunk_length"] = cropped_chunk_length
+ yield new_key, pa.Table.from_batches(chunks_buffer)
+ keys_buffer = [f"{key}[{cropped_chunk_length}:]"]
+ chunks_buffer = [chunk.slice(cropped_chunk_length, len(chunk) - cropped_chunk_length)]
+ chunks_buffer_size = len(chunk) - cropped_chunk_length
+ if self._state_dict:
+ self._state_dict["previous_state"] = previous_state
+ self._state_dict["num_chunks_since_previous_state"] = num_chunks_since_previous_state
+ if self._state_dict:
+ previous_state = self.ex_iterable.state_dict()
+ if not self.drop_last_batch and chunks_buffer:
+ new_key = "_".join(str(_key) for _key in keys_buffer)
+ if self._state_dict:
+ self._state_dict["previous_state"] = previous_state
+ self._state_dict["batch_idx"] += 1
+ self._state_dict["num_chunks_since_previous_state"] = 0
+ self._state_dict["cropped_chunk_length"] = 0
+ yield new_key, pa.Table.from_batches(chunks_buffer)
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "RebatchedArrowExamplesIterable":
+ return RebatchedArrowExamplesIterable(
+ self.ex_iterable.shuffle_data_sources(generator),
+ self.batch_size,
+ self.drop_last_batch,
+ self.force_convert_to_arrow,
+ )
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "RebatchedArrowExamplesIterable":
+ return RebatchedArrowExamplesIterable(
+ self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous),
+ self.batch_size,
+ self.drop_last_batch,
+ self.force_convert_to_arrow,
+ )
+
+ def reshard_data_sources(self) -> "RebatchedArrowExamplesIterable":
+ return RebatchedArrowExamplesIterable(
+ self.ex_iterable.reshard_data_sources(), self.batch_size, self.drop_last_batch, self.force_convert_to_arrow
+ )
+
+ @property
+ def num_shards(self) -> int:
+ return self.ex_iterable.num_shards
+
+
+class SelectColumnsIterable(_BaseExamplesIterable):
+ def __init__(self, ex_iterable: _BaseExamplesIterable, column_names: list[str]):
+ super().__init__()
+ self.ex_iterable = ex_iterable
+ self.column_names = column_names
+
+ @property
+ def iter_arrow(self):
+ if self.ex_iterable.iter_arrow:
+ return self._iter_arrow
+
+ @property
+ def is_typed(self):
+ return self.ex_iterable.is_typed
+
+ @property
+ def features(self):
+ return self.ex_iterable.features
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = self.ex_iterable._init_state_dict()
+ return self._state_dict
+
+ def __iter__(self):
+ for idx, row in self.ex_iterable:
+ yield idx, {c: row[c] for c in self.column_names}
+
+ def _iter_arrow(self) -> Iterator[tuple[Key, pa.Table]]:
+ for idx, pa_table in self.ex_iterable.iter_arrow():
+ if len(pa_table) > 0: # empty tables have no schema
+ yield idx, pa_table.select(self.column_names)
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "SelectColumnsIterable":
+ return SelectColumnsIterable(self.ex_iterable.shuffle_data_sources(generator), self.column_names)
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "SelectColumnsIterable":
+ return SelectColumnsIterable(
+ self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous), self.column_names
+ )
+
+ def reshard_data_sources(self) -> "SelectColumnsIterable":
+ return SelectColumnsIterable(self.ex_iterable.reshard_data_sources(), self.column_names)
+
+ @property
+ def num_shards(self) -> int:
+ return self.ex_iterable.num_shards
+
+
+class StepExamplesIterable(_BaseExamplesIterable):
+ def __init__(self, ex_iterable: _BaseExamplesIterable, step: int, offset: int):
+ super().__init__()
+ self.ex_iterable = ex_iterable
+ self.step = step
+ self.offset = offset
+
+ @property
+ def iter_arrow(self):
+ return self._iter_arrow if self.ex_iterable.iter_arrow else None
+
+ @property
+ def is_typed(self):
+ return self.ex_iterable.is_typed
+
+ @property
+ def features(self):
+ return self.ex_iterable.features
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = {
+ "examples_iterable": self.ex_iterable._init_state_dict(),
+ "stepped": 0,
+ "type": self.__class__.__name__,
+ }
+ return self._state_dict
+
+ def __iter__(self):
+ ex_iterator = iter(self.ex_iterable)
+ while True:
+ batch = list(islice(ex_iterator, self.step))
+ if len(batch) > self.offset:
+ yield batch[self.offset]
+ else:
+ break
+
+ def _iter_arrow(self):
+ stepped = self._state_dict["stepped"] if self._state_dict else 0
+ for key, pa_table in self.ex_iterable.iter_arrow():
+ stepped_pa_table = pa_table.take(
+ pa.array(range((self.offset - stepped) % self.step, len(pa_table), self.step), type=pa.int64())
+ )
+ stepped = (stepped + len(pa_table)) % self.step
+ if self._state_dict:
+ self._state_dict["stepped"] = stepped
+ yield key, stepped_pa_table
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "StepExamplesIterable":
+ return StepExamplesIterable(
+ self.ex_iterable.shuffle_data_sources(generator), step=self.step, offset=self.offset
+ )
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "StepExamplesIterable":
+ return StepExamplesIterable(
+ self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous),
+ step=self.step,
+ offset=self.offset,
+ )
+
+ def reshard_data_sources(self) -> "StepExamplesIterable":
+ return StepExamplesIterable(
+ self.ex_iterable.reshard_data_sources(),
+ step=self.step,
+ offset=self.offset,
+ )
+
+ @property
+ def num_shards(self) -> int:
+ return self.ex_iterable.num_shards
+
+
+class CyclingMultiSourcesExamplesIterable(_BaseExamplesIterable):
+ def __init__(
+ self,
+ ex_iterables: list[_BaseExamplesIterable],
+ stopping_strategy: Literal[
+ "first_exhausted", "all_exhausted", "all_exhausted_without_replacement"
+ ] = "first_exhausted",
+ ):
+ super().__init__()
+ self.ex_iterables = ex_iterables
+ self.stopping_strategy = stopping_strategy
+
+ # if undersampling ("first_exhausted"), we stop as soon as one dataset is exhausted
+ # if oversampling ("all_exhausted"), we stop as soons as every dataset is exhausted, i.e as soon as every samples of every dataset has been visited at least once
+ # if sampling without replacement ("all_exhausted_without_replacement"), we stop once all samples of every dataset has been visited exactly once.
+ self.bool_strategy_func = (
+ np.all if (stopping_strategy in ("all_exhausted", "all_exhausted_without_replacement")) else np.any
+ )
+
+ @property
+ def is_typed(self):
+ return self.ex_iterables[0].is_typed
+
+ @property
+ def features(self):
+ return self.ex_iterables[0].features
+
+ @property
+ def iter_arrow(self):
+ # iterate on arrow tables if all ex_iterables can iterate
+ return self._iter_arrow if all(ex_iterable.iter_arrow for ex_iterable in self.ex_iterables) else None
+
+ def _get_indices_iterator(self):
+ # this is an infinite iterator to keep track of which iterator we want to pick examples from
+ ex_iterable_idx = self._state_dict["ex_iterable_idx"] if self._state_dict else 0
+ for next_ex_iterable_idx in islice(cycle(range(len(self.ex_iterables))), ex_iterable_idx + 1, None):
+ if self._state_dict:
+ self._state_dict["ex_iterable_idx"] = next_ex_iterable_idx
+ yield ex_iterable_idx
+ ex_iterable_idx = next_ex_iterable_idx
+
+ def _init_state_dict(self) -> dict:
+ for ex_iterable in self.ex_iterables:
+ ex_iterable._init_state_dict()
+ self._state_dict = {
+ "ex_iterable_idx": 0,
+ "previous_states": [None] * len(self.ex_iterables),
+ "is_exhausted": [False] * len(self.ex_iterables),
+ "type": self.__class__.__name__,
+ }
+ return self._state_dict
+
+ def _iter_arrow(self):
+ # we use this to buffer one example of each iterator to know if an iterator is exhausted
+ nexts = [None] * len(self.ex_iterables)
+ # because of that, we need to rewind 1 example when reloading the state dict
+ if self._state_dict:
+ for i in range(len(self.ex_iterables)):
+ if self._state_dict["previous_states"][i] is not None:
+ self.ex_iterables[i].load_state_dict(self._state_dict["previous_states"][i])
+ previous_states = [ex_iterable.state_dict() for ex_iterable in self.ex_iterables]
+ iterators = [ex_iterable.iter_arrow() for ex_iterable in self.ex_iterables]
+
+ # Pre-populate futures for next samples from each iterator using threads for prefetching
+ def fetch_next_sample(iterator):
+ return next(iterator, False)
+
+ # Use ThreadPoolExecutor to fetch next samples in parallel
+ executor = concurrent.futures.ThreadPoolExecutor(max_workers=len(self.ex_iterables))
+ futures = [executor.submit(fetch_next_sample, iterator) for iterator in iterators]
+
+ indices_iterator = self._get_indices_iterator()
+
+ is_exhausted = (
+ np.array(self._state_dict["is_exhausted"]) if self._state_dict else np.full(len(self.ex_iterables), False)
+ )
+ try:
+ for i in indices_iterator:
+ # if the stopping criteria is met, break the main for loop
+ if self.bool_strategy_func(is_exhausted):
+ break
+ # Skip exhausted iterators if we sample without replacement
+ if is_exhausted[i] and self.stopping_strategy in ["all_exhausted_without_replacement"]:
+ continue
+ # let's pick one example from the iterator at index i
+ # Resolve the future to get the current sample
+ if nexts[i] is None:
+ nexts[i] = futures[i].result()
+ if self._state_dict:
+ self._state_dict["previous_states"][i] = previous_states[i]
+ previous_states[i] = self.ex_iterables[i].state_dict()
+ futures[i] = executor.submit(fetch_next_sample, iterators[i])
+ result = nexts[i]
+ # Fetch the next sample for this iterator (prefetching)
+ nexts[i] = futures[i].result()
+ if self._state_dict:
+ self._state_dict["previous_states"][i] = previous_states[i]
+ previous_states[i] = self.ex_iterables[i].state_dict()
+
+ if nexts[i] is not False:
+ futures[i] = executor.submit(fetch_next_sample, iterators[i])
+ else:
+ # the iterator is exhausted
+ is_exhausted[i] = True
+ if self._state_dict:
+ self._state_dict["is_exhausted"][i] = True
+ # we reset it in case the stopping criteria isn't met yet
+ if self.stopping_strategy not in ["all_exhausted_without_replacement"]:
+ if self._state_dict:
+ self.ex_iterables[i]._init_state_dict()
+ previous_states[i] = self.ex_iterables[i].state_dict()
+ self._state_dict["previous_states"][i] = None
+ iterators[i] = self.ex_iterables[i].iter_arrow()
+ nexts[i] = None
+ futures[i] = executor.submit(fetch_next_sample, iterators[i])
+ if result is not False:
+ yield result
+ finally:
+ # Related to https://github.com/apache/arrow/issues/45214
+ for future in futures:
+ future.result()
+ while iterators:
+ iterator = iterators.pop()
+ del iterator
+ if any(ex_iterable.sleep_on_threads_shutdown for ex_iterable in self.ex_iterables):
+ time.sleep(config.SLEEP_TIME_ON_THREADS_SHUTDOWN)
+
+ def __iter__(self):
+ # we use this to buffer one example of each iterator to know if an iterator is exhausted
+ nexts = [None] * len(self.ex_iterables)
+ # because of that, we need to rewind 1 example when reloading the state dict
+ if self._state_dict:
+ for i in range(len(self.ex_iterables)):
+ if self._state_dict["previous_states"][i] is not None:
+ self.ex_iterables[i].load_state_dict(self._state_dict["previous_states"][i])
+ previous_states = [ex_iterable.state_dict() for ex_iterable in self.ex_iterables]
+ iterators = [iter(ex_iterable) for ex_iterable in self.ex_iterables]
+
+ # Pre-populate futures for next samples from each iterator using threads for prefetching
+ def fetch_next_sample(iterator):
+ return next(iterator, False)
+
+ # Use ThreadPoolExecutor to fetch next samples in parallel
+ executor = concurrent.futures.ThreadPoolExecutor(max_workers=len(self.ex_iterables))
+ futures = [executor.submit(fetch_next_sample, iterator) for iterator in iterators]
+
+ indices_iterator = self._get_indices_iterator()
+
+ is_exhausted = (
+ np.array(self._state_dict["is_exhausted"]) if self._state_dict else np.full(len(self.ex_iterables), False)
+ )
+ try:
+ for i in indices_iterator:
+ # if the stopping criteria is met, break the main for loop
+ if self.bool_strategy_func(is_exhausted):
+ break
+ # Skip exhausted iterators if we sample without replacement
+ if is_exhausted[i] and self.stopping_strategy in ["all_exhausted_without_replacement"]:
+ continue
+ # let's pick one example from the iterator at index i
+ # Resolve the future to get the current sample
+ if nexts[i] is None:
+ nexts[i] = futures[i].result()
+ if self._state_dict:
+ self._state_dict["previous_states"][i] = previous_states[i]
+ previous_states[i] = self.ex_iterables[i].state_dict()
+ futures[i] = executor.submit(fetch_next_sample, iterators[i])
+ result = nexts[i]
+ # Fetch the next sample for this iterator (prefetching)
+ nexts[i] = futures[i].result()
+ if self._state_dict:
+ self._state_dict["previous_states"][i] = previous_states[i]
+ previous_states[i] = self.ex_iterables[i].state_dict()
+
+ if nexts[i] is not False:
+ futures[i] = executor.submit(fetch_next_sample, iterators[i])
+ else:
+ # the iterator is exhausted
+ is_exhausted[i] = True
+ if self._state_dict:
+ self._state_dict["is_exhausted"][i] = True
+ # we reset it in case the stopping criteria isn't met yet
+ if self.stopping_strategy not in ["all_exhausted_without_replacement"]:
+ if self._state_dict:
+ self.ex_iterables[i]._init_state_dict()
+ previous_states[i] = self.ex_iterables[i].state_dict()
+ self._state_dict["previous_states"][i] = None
+ iterators[i] = iter(self.ex_iterables[i])
+ nexts[i] = None
+ futures[i] = executor.submit(fetch_next_sample, iterators[i])
+ if result is not False:
+ yield result
+ finally:
+ # Related to https://github.com/apache/arrow/issues/45214
+ for future in futures:
+ future.result()
+ while iterators:
+ iterator = iterators.pop()
+ del iterator
+ executor.shutdown(wait=True)
+ if any(ex_iterable.sleep_on_threads_shutdown for ex_iterable in self.ex_iterables):
+ time.sleep(config.SLEEP_TIME_ON_THREADS_SHUTDOWN)
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "CyclingMultiSourcesExamplesIterable":
+ """Shuffle each underlying examples iterable."""
+ ex_iterables = [ex_iterable.shuffle_data_sources(generator) for ex_iterable in self.ex_iterables]
+ return CyclingMultiSourcesExamplesIterable(ex_iterables, self.stopping_strategy)
+
+ @property
+ def num_shards(self) -> int:
+ return min(ex_iterable.num_shards for ex_iterable in self.ex_iterables) if self.ex_iterables else 0
+
+ def shard_data_sources(
+ self, num_shards: int, index: int, contiguous=True
+ ) -> "CyclingMultiSourcesExamplesIterable":
+ """Either keep only the requested shard, or propagate the request to the underlying iterable."""
+ if num_shards < self.num_shards:
+ return CyclingMultiSourcesExamplesIterable(
+ [
+ iterable.shard_data_sources(num_shards, index, contiguous=contiguous)
+ for iterable in self.ex_iterables
+ ],
+ stopping_strategy=self.stopping_strategy,
+ )
+ elif index < self.num_shards:
+ return CyclingMultiSourcesExamplesIterable(
+ [
+ iterable.shard_data_sources(self.num_shards, index, contiguous=contiguous)
+ for iterable in self.ex_iterables
+ ],
+ stopping_strategy=self.stopping_strategy,
+ )
+ else:
+ return CyclingMultiSourcesExamplesIterable(
+ [],
+ stopping_strategy=self.stopping_strategy,
+ )
+
+ def reshard_data_sources(self) -> "CyclingMultiSourcesExamplesIterable":
+ return CyclingMultiSourcesExamplesIterable(
+ [iterable.reshard_data_sources() for iterable in self.ex_iterables],
+ stopping_strategy=self.stopping_strategy,
+ )
+
+
+class VerticallyConcatenatedMultiSourcesExamplesIterable(_BaseExamplesIterable):
+ """
+ VerticallyConcatenatedMultiSourcesExamplesIterable simply chains the input iterables.
+ It doesn't require the examples iterables to always yield the same columns.
+ Instead, this is handled by the `IterableDataset` class or `FormattedExamplesIterable`.
+
+ For information, `IterableDataset` merges the features of all the datasets to concatenate into one.
+ We use `IterableDataset._resolve_features` to obtain the features of all the datasets to concatenate.
+
+ Then for each example, `IterableDataset` and `FormattedExamplesIterable` automatically fill missing columns with None.
+ This is done with `_apply_feature_types_on_example`.
+ """
+
+ def __init__(self, ex_iterables: list[_BaseExamplesIterable]):
+ super().__init__()
+ self.ex_iterables = ex_iterables
+
+ @property
+ def is_typed(self):
+ return self.ex_iterables[0].is_typed
+
+ @property
+ def features(self):
+ return self.ex_iterables[0].features
+
+ @property
+ def iter_arrow(self):
+ if all(ex_iterable.iter_arrow is not None for ex_iterable in self.ex_iterables):
+ return self._iter_arrow
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = {
+ "ex_iterable_idx": 0,
+ "ex_iterables": [ex_iterable._init_state_dict() for ex_iterable in self.ex_iterables],
+ "type": self.__class__.__name__,
+ }
+ return self._state_dict
+
+ def __iter__(self):
+ ex_iterable_idx_start = self._state_dict["ex_iterable_idx"] if self._state_dict else 0
+ for ex_iterable in islice(self.ex_iterables, ex_iterable_idx_start, None):
+ yield from ex_iterable
+ if self._state_dict:
+ self._state_dict["ex_iterable_idx"] += 1
+
+ def _iter_arrow(self):
+ ex_iterable_idx_start = self._state_dict["ex_iterable_idx"] if self._state_dict else 0
+ for ex_iterable in islice(self.ex_iterables, ex_iterable_idx_start, None):
+ yield from ex_iterable.iter_arrow()
+ if self._state_dict:
+ self._state_dict["ex_iterable_idx"] += 1
+
+ def shuffle_data_sources(
+ self, generator: np.random.Generator
+ ) -> "VerticallyConcatenatedMultiSourcesExamplesIterable":
+ """Shuffle all shards."""
+ rng = deepcopy(generator)
+ single_shard_ex_iterables = [
+ ex_iterable.shard_data_sources(num_shards=ex_iterable.num_shards, index=index)
+ for ex_iterable in self.ex_iterables
+ for index in range(ex_iterable.num_shards)
+ ]
+ rng.shuffle(single_shard_ex_iterables)
+ return VerticallyConcatenatedMultiSourcesExamplesIterable(single_shard_ex_iterables)
+
+ @property
+ def num_shards(self) -> int:
+ return sum(ex_iterable.num_shards for ex_iterable in self.ex_iterables)
+
+ def shard_data_sources(
+ self, num_shards: int, index: int, contiguous=True
+ ) -> "VerticallyConcatenatedMultiSourcesExamplesIterable":
+ """Keep only the requested shard"""
+ single_shard_ex_iterables = [
+ ex_iterable.shard_data_sources(num_shards=ex_iterable.num_shards, index=index)
+ for ex_iterable in self.ex_iterables
+ for index in range(ex_iterable.num_shards)
+ ]
+ shard_indices = self.split_shard_indices_by_worker(num_shards, index, contiguous=contiguous)
+ return VerticallyConcatenatedMultiSourcesExamplesIterable(
+ [single_shard_ex_iterables[i] for i in shard_indices]
+ )
+
+ def reshard_data_sources(self) -> "VerticallyConcatenatedMultiSourcesExamplesIterable":
+ return VerticallyConcatenatedMultiSourcesExamplesIterable(
+ [iterable.reshard_data_sources() for iterable in self.ex_iterables]
+ )
+
+
+def _check_column_names(column_names: list[str]):
+ """Check the column names to make sure they don't contain duplicates."""
+ counter = Counter(column_names)
+ if not all(count == 1 for count in counter.values()):
+ duplicated_columns = [col for col in counter if counter[col] > 1]
+ raise ValueError(
+ f"The examples iterables can't have duplicated columns but columns {duplicated_columns} are duplicated."
+ )
+
+
+class HorizontallyConcatenatedMultiSourcesExamplesIterable(_BaseExamplesIterable):
+ """
+ HorizontallyConcatenatedMultiSourcesExamplesIterable merges examples together for the input list of iterables.
+ It also checks that there are no duplicate columns (otherwise we don't know which one to keep).
+ This check is done once when yielding the first example.
+
+ However it doesn't fill missing columns with None.
+ Instead, this is handled by the `IterableDataset` class or `FormattedExamplesIterable`.
+
+ For information, `IterableDataset` merges the features of all the datasets to concatenate into one.
+ We use `IterableDataset._resolve_features` to obtain the features of all the datasets to concatenate.
+
+ Then for each example, `IterableDataset` and `FormattedExamplesIterable` automatically fill missing columns with None.
+ This is done with `_apply_feature_types_on_example`.
+ """
+
+ def __init__(self, ex_iterables: list[_BaseExamplesIterable]):
+ super().__init__()
+ self.ex_iterables = ex_iterables
+
+ @property
+ def iter_arrow(self):
+ return (
+ self._iter_arrow
+ if all(
+ isinstance(ex_iterable, RebatchedArrowExamplesIterable) and ex_iterable.ex_iterable.iter_arrow
+ for ex_iterable in self.ex_iterables
+ )
+ or (len(self.ex_iterables) < 2 and all(ex_iterable.iter_arrow for ex_iterable in self.ex_iterables))
+ else None
+ )
+
+ @property
+ def is_typed(self):
+ return self.ex_iterables[0].is_typed
+
+ @property
+ def features(self):
+ return self.ex_iterables[0].features
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = {
+ "ex_iterables": [ex_iterable._init_state_dict() for ex_iterable in self.ex_iterables],
+ "type": self.__class__.__name__,
+ }
+ return self._state_dict
+
+ def __iter__(self):
+ ex_iterators = [iter(ex_iterable) for ex_iterable in self.ex_iterables]
+ for i in itertools.count():
+ keys = []
+ examples = []
+ for ex_iterator in list(ex_iterators):
+ try:
+ key, example = next(ex_iterator)
+ keys.append(key)
+ examples.append(example)
+ except StopIteration:
+ ex_iterators.remove(ex_iterator)
+ if ex_iterators:
+ if i == 0:
+ _check_column_names([column_name for example in examples for column_name in example])
+ new_example = {}
+ for example in examples:
+ new_example.update(example)
+ new_key = "_".join(str(key) for key in keys)
+ yield new_key, new_example
+ else:
+ break
+
+ def _iter_arrow(self):
+ pa_table_iterators = [iter(ex_iterable.iter_arrow()) for ex_iterable in self.ex_iterables]
+ for i in itertools.count():
+ keys = []
+ pa_tables = []
+ for pa_table_iterator in list(pa_table_iterators):
+ try:
+ key, pa_table = next(pa_table_iterator)
+ keys.append(key)
+ pa_tables.append(pa_table)
+ except StopIteration:
+ pa_table_iterators.remove(pa_table_iterator)
+ if pa_table_iterators:
+ if i == 0:
+ _check_column_names(
+ [column_name for pa_table in pa_tables for column_name in pa_table.column_names]
+ )
+ for j, table in enumerate(pa_tables):
+ if j == 0:
+ new_pa_table = table
+ else:
+ for name, col in zip(table.column_names, table.columns):
+ new_pa_table = pa_table.append_column(name, col)
+ new_key = "_".join(str(key) for key in keys)
+ yield new_key, new_pa_table
+ else:
+ break
+
+ def shuffle_data_sources(
+ self, generator: np.random.Generator
+ ) -> "HorizontallyConcatenatedMultiSourcesExamplesIterable":
+ """Doesn't shuffle the wrapped examples iterable since it would break the alignment between them."""
+ return self
+
+ @property
+ def num_shards(self) -> int:
+ return 1
+
+ def shard_data_sources(
+ self, num_shards: int, index: int, contiguous=True
+ ) -> "HorizontallyConcatenatedMultiSourcesExamplesIterable":
+ """Doesn't shard the wrapped examples iterable since it would break the alignment between them."""
+ return self
+
+ def reshard_data_sources(self) -> "HorizontallyConcatenatedMultiSourcesExamplesIterable":
+ """Doesn't reshard the wrapped examples iterable since it would break the alignment between them."""
+ return self
+
+
+class RandomlyCyclingMultiSourcesExamplesIterable(CyclingMultiSourcesExamplesIterable):
+ def __init__(
+ self,
+ ex_iterables: list[_BaseExamplesIterable],
+ generator: np.random.Generator,
+ probabilities: Optional[list[float]] = None,
+ stopping_strategy: Literal[
+ "first_exhausted", "all_exhausted", "all_exhausted_without_replacement"
+ ] = "first_exhausted",
+ ):
+ super().__init__(ex_iterables, stopping_strategy)
+ self.generator = deepcopy(generator)
+ self.probabilities = probabilities
+
+ def shift_rngs(self, value: int) -> "_BaseExamplesIterable":
+ rng = deepcopy(self.generator)
+ new_seed = rng.integers(0, 1 << 63) - value
+ return RandomlyCyclingMultiSourcesExamplesIterable(
+ ex_iterables=self.ex_iterables,
+ generator=np.random.default_rng(seed=new_seed),
+ probabilities=self.probabilities,
+ stopping_strategy=self.stopping_strategy,
+ )
+
+ @property
+ def is_typed(self):
+ return self.ex_iterables[0].is_typed
+
+ @property
+ def features(self):
+ return self.ex_iterables[0].features
+
+ def _get_indices_iterator(self):
+ rng = deepcopy(self.generator)
+ num_sources = len(self.ex_iterables)
+ random_batch_size = 1000
+ # this is an infinite iterator that randomly samples the index of the source to pick examples from
+ index_offset = self._state_dict["bit_generator_index_offset"] if self._state_dict else 0
+ if self._state_dict:
+ rng.bit_generator.state = self._state_dict["bit_generator_state"]
+ if self.probabilities is None:
+ while True:
+ for i in islice(rng.integers(0, num_sources, size=random_batch_size), index_offset, None):
+ index_offset = (index_offset + 1) % random_batch_size
+ if self._state_dict:
+ self._state_dict["bit_generator_index_offset"] = index_offset
+ if index_offset == 0:
+ self._state_dict["bit_generator_state"] = rng.bit_generator.state
+ yield int(i)
+ else:
+ while True:
+ for i in islice(
+ rng.choice(num_sources, size=random_batch_size, p=self.probabilities), index_offset, None
+ ):
+ index_offset = (index_offset + 1) % random_batch_size
+ if self._state_dict:
+ self._state_dict["bit_generator_index_offset"] = index_offset
+ if index_offset == 0:
+ self._state_dict["bit_generator_state"] = rng.bit_generator.state
+ yield int(i)
+
+ def _init_state_dict(self) -> dict:
+ for ex_iterable in self.ex_iterables:
+ ex_iterable._init_state_dict()
+ self._state_dict = {
+ "bit_generator_state": self.generator.bit_generator.state,
+ "bit_generator_index_offset": 0,
+ "previous_states": [None] * len(self.ex_iterables),
+ "is_exhausted": [False] * len(self.ex_iterables),
+ "type": self.__class__.__name__,
+ }
+ return self._state_dict
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "RandomlyCyclingMultiSourcesExamplesIterable":
+ """Shuffle the data sources of each wrapped examples iterable."""
+ ex_iterables = [ex_iterable.shuffle_data_sources(generator) for ex_iterable in self.ex_iterables]
+ return RandomlyCyclingMultiSourcesExamplesIterable(
+ ex_iterables,
+ generator=generator,
+ probabilities=self.probabilities,
+ stopping_strategy=self.stopping_strategy,
+ )
+
+ def shard_data_sources(
+ self, num_shards: int, index: int, contiguous=True
+ ) -> "RandomlyCyclingMultiSourcesExamplesIterable":
+ """Either keep only the requested shard, or propagate the request to the underlying iterable."""
+ if num_shards < self.num_shards:
+ return RandomlyCyclingMultiSourcesExamplesIterable(
+ [
+ iterable.shard_data_sources(num_shards, index, contiguous=contiguous)
+ for iterable in self.ex_iterables
+ ],
+ self.generator,
+ self.probabilities,
+ self.stopping_strategy,
+ )
+ elif index < self.num_shards:
+ return RandomlyCyclingMultiSourcesExamplesIterable(
+ [
+ iterable.shard_data_sources(self.num_shards, index, contiguous=contiguous)
+ for iterable in self.ex_iterables
+ ],
+ self.generator,
+ self.probabilities,
+ self.stopping_strategy,
+ )
+ else:
+ return RandomlyCyclingMultiSourcesExamplesIterable(
+ [],
+ self.generator,
+ self.probabilities,
+ self.stopping_strategy,
+ )
+
+ def reshard_data_sources(self) -> "RandomlyCyclingMultiSourcesExamplesIterable":
+ """Either keep only the requested shard, or propagate the request to the underlying iterable."""
+ return RandomlyCyclingMultiSourcesExamplesIterable(
+ [iterable.reshard_data_sources() for iterable in self.ex_iterables],
+ self.generator,
+ self.probabilities,
+ self.stopping_strategy,
+ )
+
+
+def _table_output_to_arrow(output) -> pa.Table:
+ if isinstance(output, pa.Table):
+ return output
+ if isinstance(output, (pd.DataFrame, pd.Series)):
+ return pa.Table.from_pandas(output)
+ if config.POLARS_AVAILABLE and "polars" in sys.modules:
+ import polars as pl
+
+ if isinstance(output, (pl.DataFrame, pl.Series)):
+ return output.to_arrow()
+ return output
+
+
+class MappedExamplesIterable(_BaseExamplesIterable):
+ def __init__(
+ self,
+ ex_iterable: _BaseExamplesIterable,
+ function: Callable,
+ with_indices: bool = False,
+ input_columns: Optional[list[str]] = None,
+ batched: bool = False,
+ batch_size: Optional[int] = 1000,
+ drop_last_batch: bool = False,
+ remove_columns: Optional[list[str]] = None,
+ fn_kwargs: Optional[dict] = None,
+ formatting: Optional["FormattingConfig"] = None,
+ features: Optional[Features] = None,
+ max_num_running_async_map_functions_in_parallel: Optional[int] = None,
+ is_batch_accumulate_arrow_table_function: bool = False,
+ ):
+ super().__init__()
+ self.ex_iterable = ex_iterable
+ self.function = function
+ self.batched = batched
+ self.batch_size = batch_size
+ self.drop_last_batch = drop_last_batch
+ self.remove_columns = remove_columns
+ self.with_indices = with_indices
+ self.input_columns = input_columns
+ self.fn_kwargs = fn_kwargs or {}
+ self.formatting = formatting # required for iter_arrow
+ self._features = features
+ self.max_num_running_async_map_functions_in_parallel = (
+ max_num_running_async_map_functions_in_parallel or config.MAX_NUM_RUNNING_ASYNC_MAP_FUNCTIONS_IN_PARALLEL
+ )
+ self.is_batch_accumulate_arrow_table_function = is_batch_accumulate_arrow_table_function
+ # sanity checks
+ if formatting and formatting.is_table:
+ # batch_size should match for iter_arrow
+ if not isinstance(ex_iterable, RebatchedArrowExamplesIterable):
+ raise ValueError(
+ f"The {formatting.format_type.capitalize()}-formatted {type(self).__name__} has underlying iterable "
+ f"that is a {type(ex_iterable).__name__} instead of a RebatchedArrowExamplesIterable."
+ )
+ elif not ex_iterable.iter_arrow:
+ raise ValueError(
+ f"The {formatting.format_type.capitalize()}-formatted {type(self).__name__} has underlying iterable "
+ f"that is a {type(ex_iterable).__name__} but doesnt' implement iter_arrow(), a possible fix could be "
+ "to use RebatchedArrowExamplesIterable(..., force_convert_to_arrow=True)."
+ )
+ elif ex_iterable.batch_size != (batch_size if batched else 1):
+ raise ValueError(
+ f"The {formatting.format_type.capitalize()}-formatted {type(self).__name__} has batch_size={batch_size if batched else 1} which is "
+ f"different from {ex_iterable.batch_size=} from its underlying iterable."
+ )
+ # to enable graceful ends
+ self._owned_loops_and_tasks: list[tuple[asyncio.AbstractEventLoop, list[asyncio.Task]]] = []
+
+ @property
+ def iter_arrow(self):
+ if self.formatting and self.formatting.is_table:
+ return self._iter_arrow
+
+ @property
+ def is_typed(self):
+ return self.features is not None # user has extracted features
+
+ @property
+ def features(self):
+ return self._features
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = {
+ "examples_iterable": self.ex_iterable._init_state_dict(),
+ "previous_state": None,
+ "num_examples_since_previous_state": 0,
+ "previous_state_example_idx": 0,
+ "type": self.__class__.__name__,
+ }
+ return self._state_dict
+
+ def __iter__(self):
+ if self.formatting and self.formatting.is_table:
+ formatter = PythonFormatter()
+ for key, pa_table in self._iter_arrow(max_chunksize=1):
+ yield key, formatter.format_row(pa_table)
+ else:
+ yield from self._iter()
+
+ def _iter(self):
+ current_idx = self._state_dict["previous_state_example_idx"] if self._state_dict else 0
+ if self._state_dict and self._state_dict["previous_state"]:
+ self.ex_iterable.load_state_dict(self._state_dict["previous_state"])
+ num_examples_to_skip = self._state_dict["num_examples_since_previous_state"]
+ else:
+ num_examples_to_skip = 0
+ iterator = iter(self.ex_iterable)
+
+ # We use the same logic as in Dataset.map, but with less features/formatting
+ # since they're handled by FormattedExamplesIterable
+
+ if self.formatting:
+ formatter = get_formatter(self.formatting.format_type)
+ format_dict = formatter.recursive_tensorize if isinstance(formatter, TensorFormatter) else None
+ else:
+ format_dict = None
+
+ def iter_batched_inputs():
+ nonlocal current_idx
+ for key, example in iterator:
+ # If `batched`, first build the batch, if `batch_size` is None or <=0, then the batch is the whole dataset
+ iterator_batch = (
+ iterator
+ if self.batch_size is None or self.batch_size <= 0
+ else islice(iterator, self.batch_size - 1)
+ )
+ key_examples_list = [(key, example)] + list(iterator_batch)
+ keys, examples = zip(*key_examples_list)
+ # the new key is the concatenation of the examples keys from the batch
+ key = "_".join(str(key) for key in keys)
+ if (
+ self.drop_last_batch
+ and self.batch_size is not None
+ and self.batch_size > 0
+ and len(examples) < self.batch_size
+ ): # ignore last batch
+ return
+ batch = _examples_to_batch(examples)
+ # we need to format here in case we need to stack tensors together
+ batch = format_dict(batch) if format_dict else batch
+ indices = [current_idx + i for i in range(len(key_examples_list))]
+ current_idx += len(indices)
+ yield indices, (key, batch)
+
+ def iter_inputs():
+ nonlocal current_idx
+ for key, example in iterator:
+ # If not batched, we can apply the transform and yield the example directly
+ # first copy the example, since we might drop some keys
+ example = dict(example)
+ # no need to do formatting here
+ current_idx += 1
+ yield current_idx - 1, (key, example)
+
+ def validate_function_output(processed_inputs):
+ if self.batched and processed_inputs:
+ first_col = next(iter(processed_inputs))
+ bad_cols = [
+ col for col in processed_inputs if len(processed_inputs[col]) != len(processed_inputs[first_col])
+ ]
+ if bad_cols:
+ raise ValueError(
+ f"Column lengths mismatch: columns {bad_cols} have length {[len(processed_inputs[col]) for col in bad_cols]} "
+ f"while {first_col} has length {len(processed_inputs[first_col])}."
+ )
+
+ def prepare_inputs(key_example, indices):
+ key, example = key_example
+ fn_args = [example] if self.input_columns is None else [example[col] for col in self.input_columns]
+ additional_args = ()
+ if self.with_indices:
+ fn_args += (indices,)
+ inputs = dict(example)
+ return inputs, fn_args, additional_args, self.fn_kwargs
+
+ def prepare_outputs(key_example, inputs, processed_inputs):
+ validate_function_output(processed_inputs)
+ # this logic mimics the one in Dataset.map
+ if self.remove_columns:
+ for c in self.remove_columns:
+ if c in inputs:
+ del inputs[c]
+ if processed_inputs is key_example[1] and c in processed_inputs:
+ del processed_inputs[c]
+ transformed_inputs = {**inputs, **processed_inputs}
+ # no need to do features decoding here
+ return transformed_inputs
+
+ def apply_function(key_example, indices):
+ """Utility to apply the function on a selection of columns."""
+ inputs, fn_args, additional_args, fn_kwargs = prepare_inputs(key_example, indices)
+ processed_inputs = self.function(*fn_args, *additional_args, **fn_kwargs)
+ return prepare_outputs(key_example, inputs, processed_inputs)
+
+ async def async_apply_function(key_example, indices):
+ """Utility to apply the function on a selection of columns. Same code but async"""
+ inputs, fn_args, additional_args, fn_kwargs = prepare_inputs(key_example, indices)
+ processed_inputs = await self.function(*fn_args, *additional_args, **fn_kwargs)
+ return prepare_outputs(key_example, inputs, processed_inputs)
+
+ tasks: list[asyncio.Task] = []
+ if inspect.iscoroutinefunction(self.function):
+ try:
+ loop = asyncio.get_running_loop()
+ except RuntimeError:
+ loop = asyncio.new_event_loop()
+ self._owned_loops_and_tasks.append((loop, tasks))
+ else:
+ loop = None
+
+ def iter_outputs():
+ nonlocal tasks, loop
+ inputs_iterator = iter_batched_inputs() if self.batched else iter_inputs()
+ if inspect.iscoroutinefunction(self.function):
+ if self._state_dict:
+ previous_state = self.ex_iterable.state_dict()
+ self._state_dict["previous_state"] = previous_state
+ previous_state_task = None
+ previous_state_example_idx = self._state_dict["previous_state_example_idx"]
+ indices: Union[list[int], list[list[int]]] = []
+ for i, key_example in inputs_iterator:
+ indices.append(i)
+ tasks.append(loop.create_task(async_apply_function(key_example, i)))
+ # keep the total active tasks under a certain number
+ if len(tasks) >= self.max_num_running_async_map_functions_in_parallel:
+ done, pending = loop.run_until_complete(
+ asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+ )
+ while tasks and len(pending) >= self.max_num_running_async_map_functions_in_parallel:
+ done, pending = loop.run_until_complete(
+ asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+ )
+ if len(tasks) >= 10 * self.max_num_running_async_map_functions_in_parallel:
+ loop.run_until_complete(tasks[0])
+ # yield finished tasks
+ while tasks and tasks[0].done():
+ i, task = indices.pop(0), tasks.pop(0)
+ yield i, task.result()
+ if self._state_dict and task is previous_state_task:
+ self._state_dict["previous_state"] = previous_state
+ self._state_dict["num_examples_since_previous_state"] = 0
+ self._state_dict["previous_state_example_idx"] = previous_state_example_idx
+ previous_state, previous_state_task = None, None
+ # checkpoint
+ if self._state_dict and previous_state_task is None and tasks:
+ previous_state = self.ex_iterable.state_dict()
+ previous_state_task = tasks[-1]
+ previous_state_example_idx = current_idx
+ while tasks:
+ yield indices[0], loop.run_until_complete(tasks[0])
+ indices.pop(0), tasks.pop(0)
+ else:
+ if self._state_dict:
+ if self.batched:
+ self._state_dict["previous_state"] = self.ex_iterable.state_dict()
+ self._state_dict["num_examples_since_previous_state"] = 0
+ self._state_dict["previous_state_example_idx"] = current_idx
+ for i, key_example in inputs_iterator:
+ if self._state_dict:
+ if not self.batched:
+ self._state_dict["previous_state_example_idx"] = current_idx
+ yield i, apply_function(key_example, i)
+ if self._state_dict:
+ if self.batched:
+ self._state_dict["previous_state"] = self.ex_iterable.state_dict()
+ self._state_dict["num_examples_since_previous_state"] = 0
+ self._state_dict["previous_state_example_idx"] = current_idx
+
+ try:
+ outputs = iter_outputs()
+ if self.batched:
+ outputs = (
+ (key, transformed_example)
+ for key, transformed_batch in outputs
+ for transformed_example in _batch_to_examples(transformed_batch)
+ )
+ for key, transformed_example in outputs:
+ if self._state_dict and self._state_dict["previous_state"] is not None:
+ self._state_dict["num_examples_since_previous_state"] += 1
+ if num_examples_to_skip > 0:
+ num_examples_to_skip -= 1
+ continue
+ yield key, transformed_example
+ except (Exception, KeyboardInterrupt):
+ if loop:
+ logger.debug(f"Canceling {len(tasks)} async tasks.")
+ for task in tasks:
+ task.cancel(msg="KeyboardInterrupt")
+ try:
+ loop.run_until_complete(asyncio.gather(*tasks))
+ except (asyncio.CancelledError, ValueError):
+ logger.debug("Tasks canceled.")
+ raise
+
+ def _iter_arrow(self, max_chunksize: Optional[int] = None) -> Iterator[tuple[Key, pa.Table]]:
+ formatter: TableFormatter = get_formatter(self.formatting.format_type) if self.formatting else ArrowFormatter()
+ if self.ex_iterable.iter_arrow:
+ iterator = self.ex_iterable.iter_arrow()
+ else:
+ iterator = _convert_to_arrow(
+ self.ex_iterable,
+ batch_size=self.batch_size if self.batched else 1,
+ drop_last_batch=self.drop_last_batch,
+ )
+ if self._state_dict and self._state_dict["previous_state"]:
+ self.ex_iterable.load_state_dict(self._state_dict["previous_state"])
+ num_examples_to_skip = self._state_dict["num_examples_since_previous_state"]
+ else:
+ num_examples_to_skip = 0
+ if self._state_dict and max_chunksize is not None:
+ self._state_dict["previous_state"] = self.ex_iterable.state_dict()
+ self._state_dict["num_examples_since_previous_state"] = 0
+ current_idx = self._state_dict["previous_state_example_idx"] if self._state_dict else 0
+ fn_kwargs = self.fn_kwargs.copy()
+ if self.is_batch_accumulate_arrow_table_function:
+ tables_accumulator: list[pa.Table] = []
+ length: Optional[int] = None
+ fn_kwargs["tables_accumulator"] = tables_accumulator
+ fn_kwargs["length"] = length
+ for key, pa_table in iterator:
+ if (
+ self.batched
+ and self.batch_size is not None
+ and len(pa_table) < self.batch_size
+ and self.drop_last_batch
+ ):
+ return
+ # first build the batch
+ function_args = (
+ [formatter.format_batch(pa_table)]
+ if self.input_columns is None
+ else [pa_table[col] for col in self.input_columns]
+ )
+ if self.with_indices:
+ if self.batched:
+ function_args.append([current_idx + i for i in range(len(pa_table))])
+ else:
+ function_args.append(current_idx)
+ # then apply the transform
+ output = self.function(*function_args, **fn_kwargs)
+ output_table = _table_output_to_arrow(output)
+ if not isinstance(output_table, pa.Table):
+ raise TypeError(
+ f"Provided `function` which is applied to {formatter.table_type} returns a variable of type "
+ f"{type(output)}. Make sure provided `function` returns a {formatter.table_type} to update the dataset."
+ )
+ # we don't need to merge results for consistency with Dataset.map which merges iif both input and output are dicts
+ # then remove the unwanted columns
+ if self.remove_columns:
+ for column in self.remove_columns:
+ if column in output_table.column_names:
+ output_table = output_table.remove_column(output_table.column_names.index(column))
+ # return output
+ if max_chunksize is None:
+ current_idx += len(pa_table)
+ if self._state_dict:
+ self._state_dict["previous_state_example_idx"] += len(pa_table)
+ yield key, output_table
+ else:
+ for i, pa_subtable in enumerate(output_table.to_reader(max_chunksize=max_chunksize)):
+ current_idx += 1
+ if self._state_dict:
+ self._state_dict["num_examples_since_previous_state"] += 1
+ if num_examples_to_skip > 0:
+ num_examples_to_skip -= 1
+ continue
+ yield f"{key}_{i}", pa_subtable
+ if self._state_dict:
+ self._state_dict["previous_state"] = self.ex_iterable.state_dict()
+ self._state_dict["num_examples_since_previous_state"] = 0
+ self._state_dict["previous_state_example_idx"] = current_idx
+ if self.is_batch_accumulate_arrow_table_function and tables_accumulator:
+ pa_table = tables_accumulator.pop(-1)
+ indices = [current_idx + i for i in range(len(pa_table))]
+ function_args = (pa_table, indices)
+ output_table = self.function(
+ *function_args, **self.fn_kwargs, tables_accumulator=tables_accumulator, length=indices[-1] + 1
+ )
+ yield "last_batch_from_tables_accumulator", output_table
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "MappedExamplesIterable":
+ """Shuffle the wrapped examples iterable."""
+ return MappedExamplesIterable(
+ self.ex_iterable.shuffle_data_sources(generator),
+ function=self.function,
+ with_indices=self.with_indices,
+ input_columns=self.input_columns,
+ batched=self.batched,
+ batch_size=self.batch_size,
+ drop_last_batch=self.drop_last_batch,
+ remove_columns=self.remove_columns,
+ fn_kwargs=self.fn_kwargs,
+ formatting=self.formatting,
+ features=self.features,
+ max_num_running_async_map_functions_in_parallel=self.max_num_running_async_map_functions_in_parallel,
+ )
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "MappedExamplesIterable":
+ """Keep only the requested shard."""
+ return MappedExamplesIterable(
+ self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous),
+ function=self.function,
+ with_indices=self.with_indices,
+ input_columns=self.input_columns,
+ batched=self.batched,
+ batch_size=self.batch_size,
+ drop_last_batch=self.drop_last_batch,
+ remove_columns=self.remove_columns,
+ fn_kwargs=self.fn_kwargs,
+ formatting=self.formatting,
+ features=self.features,
+ max_num_running_async_map_functions_in_parallel=self.max_num_running_async_map_functions_in_parallel,
+ )
+
+ def reshard_data_sources(self) -> "MappedExamplesIterable":
+ return MappedExamplesIterable(
+ self.ex_iterable.reshard_data_sources(),
+ function=self.function,
+ with_indices=self.with_indices,
+ input_columns=self.input_columns,
+ batched=self.batched,
+ batch_size=self.batch_size,
+ drop_last_batch=self.drop_last_batch,
+ remove_columns=self.remove_columns,
+ fn_kwargs=self.fn_kwargs,
+ formatting=self.formatting,
+ features=self.features,
+ max_num_running_async_map_functions_in_parallel=self.max_num_running_async_map_functions_in_parallel,
+ )
+
+ @property
+ def num_shards(self) -> int:
+ return self.ex_iterable.num_shards
+
+
+def _add_mask(
+ input: Union[dict, pa.Table],
+ mask: Union[bool, list, pa.Array, pa.ChunkedArray, pa.BooleanScalar],
+ mask_column_name: str,
+):
+ if isinstance(input, pa.Table):
+ if not isinstance(mask, (list, pa.Array, pa.ChunkedArray)):
+ mask = pa.array([mask], type=pa.bool_())
+ return input.append_column(mask_column_name, mask)
+ else:
+ return {mask_column_name: mask}
+
+
+def add_mask(mask_function: Callable, input: Union[dict, pa.Table], *args, mask_column_name: str, **kwargs):
+ mask = mask_function(input, *args, **kwargs)
+ return _add_mask(input, mask, mask_column_name)
+
+
+async def async_add_mask(
+ mask_function: Callable, input: Union[dict, pa.Table], *args, mask_column_name: str, **kwargs
+):
+ mask = await mask_function(input, *args, **kwargs)
+ return _add_mask(input, mask, mask_column_name)
+
+
+class FilteredExamplesIterable(MappedExamplesIterable):
+ mask_column_name = "===MASK==="
+
+ def __init__(
+ self,
+ ex_iterable: _BaseExamplesIterable,
+ function: Callable,
+ with_indices: bool = False,
+ input_columns: Optional[list[str]] = None,
+ batched: bool = False,
+ batch_size: Optional[int] = 1000,
+ fn_kwargs: Optional[dict] = None,
+ formatting: Optional["FormattingConfig"] = None,
+ ):
+ self.mask_function = function
+ if ex_iterable.is_typed:
+ features = Features({**ex_iterable.features, self.mask_column_name: Value("bool")})
+ else:
+ features = None
+ super().__init__(
+ ex_iterable=ex_iterable,
+ function=partial(
+ async_add_mask if inspect.iscoroutinefunction(function) else add_mask,
+ function,
+ mask_column_name=self.mask_column_name,
+ ),
+ with_indices=with_indices,
+ input_columns=input_columns,
+ batched=batched,
+ batch_size=batch_size,
+ fn_kwargs=fn_kwargs,
+ formatting=formatting,
+ features=features,
+ )
+
+ def _iter(self):
+ for key, example in super()._iter():
+ example = dict(example)
+ if example.pop(self.mask_column_name):
+ yield key, example
+
+ def _iter_arrow(self, max_chunksize: Optional[int] = None):
+ for key, pa_table in super()._iter_arrow(max_chunksize=max_chunksize):
+ mask = pa_table[self.mask_column_name]
+ yield key, pa_table.drop(self.mask_column_name).filter(mask)
+
+ def shuffle_data_sources(self, seed: Optional[int]) -> "FilteredExamplesIterable":
+ """Shuffle the wrapped examples iterable."""
+ return FilteredExamplesIterable(
+ self.ex_iterable.shuffle_data_sources(seed),
+ function=self.mask_function,
+ with_indices=self.with_indices,
+ input_columns=self.input_columns,
+ batched=self.batched,
+ batch_size=self.batch_size,
+ fn_kwargs=self.fn_kwargs,
+ formatting=self.formatting,
+ )
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "FilteredExamplesIterable":
+ """Keep only the requested shard."""
+ return FilteredExamplesIterable(
+ self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous),
+ function=self.mask_function,
+ with_indices=self.with_indices,
+ input_columns=self.input_columns,
+ batched=self.batched,
+ batch_size=self.batch_size,
+ fn_kwargs=self.fn_kwargs,
+ formatting=self.formatting,
+ )
+
+ def reshard_data_sources(self) -> "FilteredExamplesIterable":
+ return FilteredExamplesIterable(
+ self.ex_iterable.reshard_data_sources(),
+ function=self.mask_function,
+ with_indices=self.with_indices,
+ input_columns=self.input_columns,
+ batched=self.batched,
+ batch_size=self.batch_size,
+ fn_kwargs=self.fn_kwargs,
+ formatting=self.formatting,
+ )
+
+ @property
+ def num_shards(self) -> int:
+ return self.ex_iterable.num_shards
+
+
+class BufferShuffledExamplesIterable(_BaseExamplesIterable):
+ def __init__(self, ex_iterable: _BaseExamplesIterable, buffer_size: int, generator: np.random.Generator):
+ super().__init__()
+ self.ex_iterable = ex_iterable
+ self.buffer_size = buffer_size
+ self.generator = generator
+
+ def shift_rngs(self, value: int) -> "_BaseExamplesIterable":
+ rng = deepcopy(self.generator)
+ new_seed = rng.integers(0, 1 << 63) - value
+ return BufferShuffledExamplesIterable(
+ ex_iterable=self.ex_iterable,
+ buffer_size=self.buffer_size,
+ generator=np.random.default_rng(seed=new_seed),
+ )
+
+ @property
+ def is_typed(self):
+ return self.ex_iterable.is_typed
+
+ @property
+ def features(self):
+ return self.ex_iterable.features
+
+ @property
+ def iter_arrow(self):
+ return self._iter_arrow if self.ex_iterable.iter_arrow else None
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = self.ex_iterable._init_state_dict()
+ self._original_state_dict = self.state_dict()
+ return self._state_dict
+
+ def load_state_dict(self, state_dict: dict) -> dict:
+ if self._state_dict:
+ if state_dict != self._original_state_dict:
+ logger.warning(
+ "Loading a state dict of a shuffle buffer of a dataset without the buffer content."
+ "The shuffle buffer will be refilled before starting to yield new examples."
+ )
+ return super().load_state_dict(state_dict)
+
+ @staticmethod
+ def _iter_random_indices(rng: np.random.Generator, buffer_size: int, random_batch_size=1000) -> Iterator[int]:
+ while True:
+ yield from (int(i) for i in rng.integers(0, buffer_size, size=random_batch_size))
+
+ def __iter__(self):
+ buffer_size = self.buffer_size
+ rng = deepcopy(self.generator)
+ indices_iterator = self._iter_random_indices(rng, buffer_size)
+ # this is the shuffle buffer that we keep in memory
+ mem_buffer = []
+ for x in self.ex_iterable:
+ if len(mem_buffer) == buffer_size: # if the buffer is full, pick and example from it
+ i = next(indices_iterator)
+ yield mem_buffer[i]
+ mem_buffer[i] = x # replace the picked example by a new one
+ else: # otherwise, keep filling the buffer
+ mem_buffer.append(x)
+ # when we run out of examples, we shuffle the remaining examples in the buffer and yield them
+ rng.shuffle(mem_buffer)
+ yield from mem_buffer
+
+ def _iter_arrow(self):
+ buffer_size = self.buffer_size
+ rng = deepcopy(self.generator)
+ indices_iterator = self._iter_random_indices(rng, buffer_size)
+ # this is the shuffle buffer that we keep in memory
+ mem_buffer = []
+ for key, pa_table in self.ex_iterable.iter_arrow():
+ if len(mem_buffer) == buffer_size: # if the buffer is full, pick and example from it
+ i = next(indices_iterator)
+ yield mem_buffer[i]
+ mem_buffer[i] = (key, pa_table) # replace the picked example by a new one
+ else: # otherwise, keep filling the buffer
+ mem_buffer.append((key, pa_table))
+ # when we run out of examples, we shuffle the remaining examples in the buffer and yield them
+ rng.shuffle(mem_buffer)
+ yield from mem_buffer
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "BufferShuffledExamplesIterable":
+ """Shuffle the wrapped examples iterable as well as the shuffling buffer."""
+ return BufferShuffledExamplesIterable(
+ self.ex_iterable.shuffle_data_sources(generator), buffer_size=self.buffer_size, generator=self.generator
+ )
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "BufferShuffledExamplesIterable":
+ """Keep only the requested shard."""
+ return BufferShuffledExamplesIterable(
+ self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous),
+ buffer_size=self.buffer_size,
+ generator=self.generator,
+ )
+
+ def reshard_data_sources(self) -> "BufferShuffledExamplesIterable":
+ return BufferShuffledExamplesIterable(
+ self.ex_iterable.reshard_data_sources(),
+ buffer_size=self.buffer_size,
+ generator=self.generator,
+ )
+
+ @property
+ def num_shards(self) -> int:
+ return self.ex_iterable.num_shards
+
+
+class DataSourcesShufflingDisallowed(Exception):
+ """skip() or take() freeze the order of data sources shards"""
+
+
+class SkipExamplesIterable(_BaseExamplesIterable):
+ def __init__(
+ self,
+ ex_iterable: _BaseExamplesIterable,
+ n: int,
+ block_sources_order_when_shuffling: bool = True,
+ split_when_sharding: bool = True,
+ ):
+ super().__init__()
+ self.ex_iterable = ex_iterable
+ self.n = n
+ self.block_sources_order_when_shuffling = block_sources_order_when_shuffling
+ self.split_when_sharding = split_when_sharding
+
+ @property
+ def iter_arrow(self):
+ return self._iter_arrow if self.ex_iterable.iter_arrow else None
+
+ @property
+ def is_typed(self):
+ return self.ex_iterable.is_typed
+
+ @property
+ def features(self):
+ return self.ex_iterable.features
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = {
+ "skipped": 0,
+ "examples_iterable": self.ex_iterable._init_state_dict(),
+ "type": self.__class__.__name__,
+ }
+ return self._state_dict
+
+ def __iter__(self):
+ skipped = self._state_dict["skipped"] if self._state_dict else 0
+ for key_example in self.ex_iterable:
+ if skipped + 1 <= self.n:
+ skipped += 1
+ if self._state_dict:
+ self._state_dict["skipped"] = skipped
+ else:
+ yield key_example
+
+ def _iter_arrow(self):
+ skipped = self._state_dict["skipped"] if self._state_dict else 0
+ for key, pa_table in self.ex_iterable.iter_arrow():
+ if len(pa_table) == 0:
+ continue
+ elif skipped + len(pa_table) <= self.n:
+ skipped += len(pa_table)
+ if self._state_dict:
+ self._state_dict["skipped"] = skipped
+ elif skipped + 1 <= self.n:
+ offset = self.n - skipped
+ skipped = self.n
+ if self._state_dict:
+ self._state_dict["skipped"] = skipped
+ yield key, pa_table.slice(offset, len(pa_table) - offset)
+ else:
+ yield key, pa_table
+
+ @staticmethod
+ def split_number(num, n):
+ quotient = num // n
+ remainder = num % n
+ result = [quotient] * n
+ for i in range(remainder):
+ result[i] += 1
+ return result
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "SkipExamplesIterable":
+ """May not shuffle the wrapped examples iterable since it would skip examples from other shards instead."""
+ if self.block_sources_order_when_shuffling:
+ raise DataSourcesShufflingDisallowed()
+ else:
+ return SkipExamplesIterable(
+ self.ex_iterable.shuffle_data_sources(generator),
+ n=self.n,
+ block_sources_order_when_shuffling=self.block_sources_order_when_shuffling,
+ split_when_sharding=self.split_when_sharding,
+ )
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "SkipExamplesIterable":
+ """Keep only the requested shard."""
+ if self.split_when_sharding:
+ return SkipExamplesIterable(
+ self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous),
+ n=self.split_number(self.n, num_shards)[index],
+ block_sources_order_when_shuffling=self.block_sources_order_when_shuffling,
+ split_when_sharding=self.split_when_sharding,
+ )
+ else:
+ return self
+
+ def reshard_data_sources(self) -> "SkipExamplesIterable":
+ return SkipExamplesIterable(
+ self.ex_iterable.reshard_data_sources(),
+ n=self.n,
+ block_sources_order_when_shuffling=self.block_sources_order_when_shuffling,
+ split_when_sharding=self.split_when_sharding,
+ )
+
+ @property
+ def num_shards(self) -> int:
+ return self.ex_iterable.num_shards
+
+
+class RepeatExamplesIterable(_BaseExamplesIterable):
+ """
+ Iterable that repeats the underlying iterable a given number of times.
+ """
+
+ def __init__(
+ self,
+ ex_iterable: _BaseExamplesIterable,
+ num_times: Optional[int],
+ ):
+ super().__init__()
+ self.ex_iterable = ex_iterable
+ self.num_times = num_times
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = {
+ "repeat_index": 0,
+ "examples_iterable": self.ex_iterable._init_state_dict(),
+ "type": self.__class__.__name__,
+ }
+ return self._state_dict
+
+ def __iter__(self):
+ repeat_index = self._state_dict["repeat_index"] if self._state_dict else 0
+ while True:
+ if self.num_times is not None and repeat_index >= max(self.num_times, 0):
+ break
+ yield from self.ex_iterable
+ repeat_index += 1
+ if self._state_dict:
+ self._state_dict["repeat_index"] = repeat_index
+ self._state_dict["examples_iterable"] = self.ex_iterable._init_state_dict()
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "RepeatExamplesIterable":
+ """Shuffle the underlying iterable, then repeat."""
+ return RepeatExamplesIterable(self.ex_iterable.shuffle_data_sources(generator), num_times=self.num_times)
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "RepeatExamplesIterable":
+ """Shard, then repeat shards."""
+ return RepeatExamplesIterable(
+ self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous),
+ num_times=self.num_times,
+ )
+
+ def reshard_data_sources(self) -> "RepeatExamplesIterable":
+ return RepeatExamplesIterable(
+ self.ex_iterable.reshard_data_sources(),
+ num_times=self.num_times,
+ )
+
+ @property
+ def num_shards(self) -> int:
+ return self.ex_iterable.num_shards
+
+
+class TakeExamplesIterable(_BaseExamplesIterable):
+ def __init__(
+ self,
+ ex_iterable: _BaseExamplesIterable,
+ n: int,
+ block_sources_order_when_shuffling: bool = True,
+ split_when_sharding: bool = True,
+ ):
+ super().__init__()
+ self.ex_iterable = ex_iterable
+ self.n = n
+ self.block_sources_order_when_shuffling = block_sources_order_when_shuffling
+ self.split_when_sharding = split_when_sharding
+
+ @property
+ def iter_arrow(self):
+ return self._iter_arrow if self.ex_iterable.iter_arrow else None
+
+ @property
+ def is_typed(self):
+ return self.ex_iterable.is_typed
+
+ @property
+ def features(self):
+ return self.ex_iterable.features
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = {
+ "taken": 0,
+ "examples_iterable": self.ex_iterable._init_state_dict(),
+ "type": self.__class__.__name__,
+ }
+ return self._state_dict
+
+ def __iter__(self):
+ taken = self._state_dict["taken"] if self._state_dict else 0
+ if taken >= self.n:
+ return
+ for key_example in self.ex_iterable:
+ if taken + 1 <= self.n:
+ taken += 1
+ if self._state_dict:
+ self._state_dict["taken"] = taken
+ yield key_example
+ else:
+ break
+
+ def _iter_arrow(self):
+ taken = self._state_dict["taken"] if self._state_dict else 0
+ if taken >= self.n:
+ return
+ for key, pa_table in self.ex_iterable.iter_arrow():
+ if len(pa_table) == 0:
+ continue
+ elif taken + len(pa_table) <= self.n:
+ taken += len(pa_table)
+ if self._state_dict:
+ self._state_dict["taken"] = taken
+ yield key, pa_table
+ elif taken + 1 <= self.n:
+ length = self.n - taken
+ taken = self.n
+ if self._state_dict:
+ self._state_dict["taken"] = taken
+ yield key, pa_table.slice(0, length)
+ else:
+ break
+
+ @staticmethod
+ def split_number(num, n):
+ quotient = num // n
+ remainder = num % n
+ result = [quotient] * n
+ for i in range(remainder):
+ result[i] += 1
+ return result
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "TakeExamplesIterable":
+ """May not shuffle the wrapped examples iterable since it would take examples from other shards instead."""
+ if self.block_sources_order_when_shuffling:
+ raise DataSourcesShufflingDisallowed()
+ else:
+ return TakeExamplesIterable(
+ self.ex_iterable.shuffle_data_sources(generator),
+ n=self.n,
+ block_sources_order_when_shuffling=self.block_sources_order_when_shuffling,
+ split_when_sharding=self.split_when_sharding,
+ )
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "TakeExamplesIterable":
+ """Keep only the requested shard."""
+ if self.split_when_sharding:
+ return TakeExamplesIterable(
+ self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous),
+ n=self.split_number(self.n, num_shards)[index],
+ block_sources_order_when_shuffling=self.block_sources_order_when_shuffling,
+ split_when_sharding=self.split_when_sharding,
+ )
+ else:
+ return TakeExamplesIterable(
+ self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous),
+ n=self.n,
+ block_sources_order_when_shuffling=self.block_sources_order_when_shuffling,
+ split_when_sharding=self.split_when_sharding,
+ )
+
+ def reshard_data_sources(self) -> "TakeExamplesIterable":
+ return TakeExamplesIterable(
+ self.ex_iterable.reshard_data_sources(),
+ n=self.n,
+ block_sources_order_when_shuffling=self.block_sources_order_when_shuffling,
+ split_when_sharding=self.split_when_sharding,
+ )
+
+ @property
+ def num_shards(self) -> int:
+ return self.ex_iterable.num_shards
+
+
+def _apply_feature_types_on_example(
+ example: dict, features: Features, token_per_repo_id: dict[str, Union[str, bool, None]]
+) -> dict:
+ example = dict(example)
+ # add missing columns
+ for column_name in features:
+ if column_name not in example:
+ example[column_name] = None
+ # we encode the example for ClassLabel feature types for example
+ encoded_example = features.encode_example(example)
+ # Decode example for Audio feature, e.g.
+ decoded_example = features.decode_example(encoded_example, token_per_repo_id=token_per_repo_id)
+ return decoded_example
+
+
+@dataclass
+class FormattingConfig:
+ format_type: Optional[str]
+
+ @property
+ def is_table(self) -> bool:
+ return isinstance(get_formatter(self.format_type), TableFormatter)
+
+ @property
+ def is_tensor(self) -> bool:
+ return isinstance(get_formatter(self.format_type), TensorFormatter)
+
+
+class FormattedExamplesIterable(_BaseExamplesIterable):
+ def __init__(
+ self,
+ ex_iterable: _BaseExamplesIterable,
+ formatting: Optional[FormattingConfig],
+ features: Optional[Features],
+ token_per_repo_id: dict[str, Union[str, bool, None]],
+ force_convert_to_python: bool = False,
+ ):
+ super().__init__()
+ self.ex_iterable = ex_iterable
+ self._features = features
+ self.formatting = formatting
+ self.token_per_repo_id = token_per_repo_id
+ self.force_convert_to_python = force_convert_to_python
+
+ @property
+ def iter_arrow(self):
+ if self.ex_iterable.iter_arrow and not self.force_convert_to_python:
+ return self._iter_arrow
+
+ @property
+ def is_typed(self):
+ return self.ex_iterable.is_typed or self._features is not None
+
+ @property
+ def features(self):
+ return self._features
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = self.ex_iterable._init_state_dict()
+ return self._state_dict
+
+ def __iter__(self):
+ if not self.formatting or self.formatting.is_table:
+ formatter = PythonFormatter(
+ features=self._features if not self.ex_iterable.is_typed else None,
+ token_per_repo_id=self.token_per_repo_id,
+ )
+ else:
+ formatter = get_formatter(
+ self.formatting.format_type,
+ features=self._features if not self.ex_iterable.is_typed else None,
+ token_per_repo_id=self.token_per_repo_id,
+ )
+
+ # It's ok to use _iter_arrow here without fancy state_dict logic since it's
+ # used with RebatchedArrowExamplesIterable with the right batch_size to
+ # never lose examples
+ if self.ex_iterable.iter_arrow:
+ # feature casting (inc column addition) handled within self._iter_arrow()
+ for key, pa_table in self._iter_arrow():
+ batch = formatter.format_batch(pa_table)
+ for example in _batch_to_examples(batch):
+ yield key, example
+ else:
+ format_dict = (
+ formatter.recursive_tensorize
+ if isinstance(formatter, TensorFormatter)
+ else None # cast in case features is None
+ )
+ for key, example in self.ex_iterable:
+ # don't apply feature types if already applied by ex_iterable (e.g. in case of chained with_format)
+ if self.features and not self.ex_iterable.is_typed:
+ example = _apply_feature_types_on_example(
+ example, self.features, token_per_repo_id=self.token_per_repo_id
+ )
+ if format_dict:
+ example = format_dict(example)
+ yield key, example
+
+ def _iter_arrow(self) -> Iterator[tuple[Key, pa.Table]]:
+ if not self.features:
+ yield from self.ex_iterable._iter_arrow()
+ return
+ schema = self.features.arrow_schema
+ for key, pa_table in self.ex_iterable._iter_arrow():
+ columns = set(pa_table.column_names)
+ # add missing columns
+ for column_name in self.features:
+ if column_name not in columns:
+ col = pa.NullArray.from_buffers(pa.null(), len(pa_table), [None])
+ pa_table = pa_table.append_column(column_name, col)
+ if pa_table.schema != schema:
+ pa_table = cast_table_to_features(pa_table, self.features)
+ yield key, pa_table
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "FormattedExamplesIterable":
+ """Shuffle the wrapped examples iterable."""
+ return FormattedExamplesIterable(
+ self.ex_iterable.shuffle_data_sources(generator),
+ features=self.features,
+ token_per_repo_id=self.token_per_repo_id,
+ formatting=self.formatting,
+ force_convert_to_python=self.force_convert_to_python,
+ )
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "FormattedExamplesIterable":
+ """Keep only the requested shard."""
+ return FormattedExamplesIterable(
+ self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous),
+ features=self.features,
+ token_per_repo_id=self.token_per_repo_id,
+ formatting=self.formatting,
+ force_convert_to_python=self.force_convert_to_python,
+ )
+
+ def reshard_data_sources(self) -> "FormattedExamplesIterable":
+ return FormattedExamplesIterable(
+ self.ex_iterable.reshard_data_sources(),
+ features=self.features,
+ token_per_repo_id=self.token_per_repo_id,
+ formatting=self.formatting,
+ force_convert_to_python=self.force_convert_to_python,
+ )
+
+ @property
+ def num_shards(self) -> int:
+ return self.ex_iterable.num_shards
+
+
+@dataclass
+class DistributedConfig:
+ rank: int
+ world_size: int
+
+
+def _maybe_add_torch_iterable_dataset_parent_class(cls):
+ """Add torch.utils.data.IterableDataset as a parent class if 'torch' is available"""
+ if config.TORCH_AVAILABLE:
+ import torch.utils.data
+
+ if torch.utils.data.IterableDataset not in cls.__bases__:
+ cls.__bases__ += (torch.utils.data.IterableDataset,)
+
+
+def _maybe_share_with_torch_persistent_workers(value: Union[int, "torch.Tensor"]) -> Union[int, "torch.Tensor"]:
+ if config.TORCH_AVAILABLE:
+ import torch
+
+ if isinstance(value, torch.Tensor):
+ return value.share_memory_()
+ else:
+ return torch.tensor(value).share_memory_()
+ else:
+ return value
+
+
+class IterableColumn:
+ """
+ An iterable for a specific column of an [`IterableDataset`].
+
+ Example:
+
+ Iterate on the texts of the "text" column of a dataset:
+
+ ```python
+ for text in dataset["text"]:
+ ...
+ ```
+
+ It also works with nested columns:
+
+ ```python
+ for source in dataset["metadata"]["source"]:
+ ...
+ ```
+ """
+
+ def __init__(self, source: Union["IterableDataset", "IterableColumn"], column_name: str):
+ self.source = source
+ self.column_name = column_name
+
+ def __iter__(self) -> Iterator[Any]:
+ for example in self.source:
+ yield example[self.column_name]
+
+ def __getitem__(self, column_name: str) -> "IterableColumn":
+ return IterableColumn(self, column_name)
+
+
+class IterableDataset(DatasetInfoMixin):
+ """A Dataset backed by an iterable."""
+
+ def __init__(
+ self,
+ ex_iterable: _BaseExamplesIterable,
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ formatting: Optional[FormattingConfig] = None,
+ distributed: Optional[DistributedConfig] = None,
+ token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None,
+ ):
+ info = info.copy() if info is not None else DatasetInfo()
+ DatasetInfoMixin.__init__(self, info=info, split=split)
+
+ self._ex_iterable = copy(ex_iterable)
+ self._formatting = formatting
+ self._distributed = distributed
+ self._token_per_repo_id: dict[str, Union[str, bool, None]] = token_per_repo_id or {}
+ self._epoch: Union[int, "torch.Tensor"] = _maybe_share_with_torch_persistent_workers(0)
+ self._starting_state_dict: Optional[dict] = None
+ self.__hffs_cache = HfFileSystem._cache # keep the cache on pickling (e.g. for dataloader workers)
+ self._prepare_ex_iterable_for_iteration() # set state_dict
+ _maybe_add_torch_iterable_dataset_parent_class(self.__class__) # subclass of torch IterableDataset
+
+ @property
+ def num_columns(self) -> Optional[int]:
+ """Number of columns in the dataset.
+ This can be None if the dataset has unknown features (e.g. after a map() operation).
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+ >>> ds.num_columns
+ 2
+ ```
+ """
+ return None if self.features is None else len(self.features)
+
+ @property
+ def column_names(self) -> Optional[list[str]]:
+ """Names of the columns in the dataset.
+ This can be None if the dataset has unknown features (e.g. after a map() operation).
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation", streaming=True)
+ >>> ds.column_names
+ ['text', 'label']
+ ```
+ """
+ return None if self.features is None else list(self.features)
+
+ def state_dict(self) -> dict:
+ """Get the current state_dict of the dataset.
+ It corresponds to the state at the latest example it yielded.
+
+ Resuming returns exactly where the checkpoint was saved except in two cases:
+
+ 1. examples from shuffle buffers are lost when resuming and the buffers are refilled with new data
+ 2. combinations of `.with_format(arrow)` and batched `.map()` may skip one batch.
+
+ Returns:
+ `dict`
+
+ Example:
+
+ ```py
+ >>> from datasets import Dataset, concatenate_datasets
+ >>> ds = Dataset.from_dict({"a": range(6)}).to_iterable_dataset(num_shards=3)
+ >>> for idx, example in enumerate(ds):
+ ... print(example)
+ ... if idx == 2:
+ ... state_dict = ds.state_dict()
+ ... print("checkpoint")
+ ... break
+ >>> ds.load_state_dict(state_dict)
+ >>> print(f"restart from checkpoint")
+ >>> for example in ds:
+ ... print(example)
+ ```
+
+ which returns:
+ ```
+ {'a': 0}
+ {'a': 1}
+ {'a': 2}
+ checkpoint
+ restart from checkpoint
+ {'a': 3}
+ {'a': 4}
+ {'a': 5}
+ ```
+
+ ```py
+ >>> from torchdata.stateful_dataloader import StatefulDataLoader
+ >>> ds = load_dataset("deepmind/code_contests", streaming=True, split="train")
+ >>> dataloader = StatefulDataLoader(ds, batch_size=32, num_workers=4)
+ >>> # checkpoint
+ >>> state_dict = dataloader.state_dict() # uses ds.state_dict() under the hood
+ >>> # resume from checkpoint
+ >>> dataloader.load_state_dict(state_dict) # uses ds.load_state_dict() under the hood
+ ```
+ """
+ return deepcopy(self._state_dict)
+
+ def load_state_dict(self, state_dict: dict) -> None:
+ """Load the state_dict of the dataset.
+ The iteration will restart at the next example from when the state was saved.
+
+ Resuming returns exactly where the checkpoint was saved except in two cases:
+
+ 1. examples from shuffle buffers are lost when resuming and the buffers are refilled with new data
+ 2. combinations of `.with_format(arrow)` and batched `.map()` may skip one batch.
+
+ Example:
+
+ ```py
+ >>> from datasets import Dataset, concatenate_datasets
+ >>> ds = Dataset.from_dict({"a": range(6)}).to_iterable_dataset(num_shards=3)
+ >>> for idx, example in enumerate(ds):
+ ... print(example)
+ ... if idx == 2:
+ ... state_dict = ds.state_dict()
+ ... print("checkpoint")
+ ... break
+ >>> ds.load_state_dict(state_dict)
+ >>> print(f"restart from checkpoint")
+ >>> for example in ds:
+ ... print(example)
+ ```
+
+ which returns:
+ ```
+ {'a': 0}
+ {'a': 1}
+ {'a': 2}
+ checkpoint
+ restart from checkpoint
+ {'a': 3}
+ {'a': 4}
+ {'a': 5}
+ ```
+
+ ```py
+ >>> from torchdata.stateful_dataloader import StatefulDataLoader
+ >>> ds = load_dataset("deepmind/code_contests", streaming=True, split="train")
+ >>> dataloader = StatefulDataLoader(ds, batch_size=32, num_workers=4)
+ >>> # checkpoint
+ >>> state_dict = dataloader.state_dict() # uses ds.state_dict() under the hood
+ >>> # resume from checkpoint
+ >>> dataloader.load_state_dict(state_dict) # uses ds.load_state_dict() under the hood
+ ```
+ """
+ self._starting_state_dict = state_dict
+
+ def __repr__(self):
+ return f"IterableDataset({{\n features: {list(self._info.features.keys()) if self._info.features is not None else 'Unknown'},\n num_shards: {self.num_shards}\n}})"
+
+ def __getstate__(self):
+ return self.__dict__
+
+ def __setstate__(self, d):
+ self.__dict__ = d
+ # Re-add torch shared memory, since shared memory is not always kept when pickling
+ self._epoch = _maybe_share_with_torch_persistent_workers(self._epoch)
+ # Re-add the cache to keep on pickling (e.g. for dataloader workers)
+ self.__hffs_cache = HfFileSystem._cache
+ # Re-add torch iterable dataset as a parent class, since dynamically added parent classes are not kept when pickling
+ _maybe_add_torch_iterable_dataset_parent_class(self.__class__)
+
+ def _head(self, n=5):
+ return next(iter(self.iter(batch_size=n)))
+
+ @property
+ def epoch(self) -> int:
+ return int(self._epoch)
+
+ @property
+ def num_shards(self) -> int:
+ if self._distributed and self._ex_iterable.num_shards % self._distributed.world_size == 0:
+ return self._ex_iterable.num_shards // self._distributed.world_size
+ return self._ex_iterable.num_shards
+
+ @property
+ def n_shards(self) -> int: # backward compatibility
+ return self.num_shards
+
+ def _iter_pytorch(self):
+ ex_iterable = self._prepare_ex_iterable_for_iteration()
+ # Fix for fsspec when using multiprocess to avoid hanging in the ML training loop. (only required for fsspec >= 0.9.0)
+ # See https://github.com/fsspec/gcsfs/issues/379
+ fsspec.asyn.reset_lock()
+ # check if there aren't too many workers
+ import torch.utils.data
+
+ worker_info = torch.utils.data.get_worker_info()
+ if self._is_main_process() and ex_iterable.num_shards < worker_info.num_workers:
+ logger.warning(
+ f"Too many dataloader workers: {worker_info.num_workers} (max is dataset.num_shards={ex_iterable.num_shards}). "
+ f"Stopping {worker_info.num_workers - ex_iterable.num_shards} dataloader workers."
+ )
+ logger.info(
+ f"To parallelize data loading, we give each process some shards (or data sources) to process. "
+ f"Therefore it's unnecessary to have a number of workers greater than dataset.num_shards={ex_iterable.num_shards}. "
+ f"To enable more parallelism, please split the dataset in more files than {ex_iterable.num_shards} or try `dataset = dataset.reshard()` which may increase `num_shards` depending on the dataset file format."
+ )
+ # split workload
+ _log_prefix = f"node#{self._distributed.rank} " if self._distributed else ""
+ shards_indices = ex_iterable.split_shard_indices_by_worker(
+ num_shards=worker_info.num_workers, index=worker_info.id, contiguous=False
+ )
+ if shards_indices:
+ logger.debug(
+ f"{_log_prefix}dataloader worker#{worker_info.id}, ': Starting to iterate over {len(shards_indices)}/{ex_iterable.num_shards} shards."
+ )
+ ex_iterable = ex_iterable.shard_data_sources(
+ num_shards=worker_info.num_workers, index=worker_info.id, contiguous=False
+ )
+ ex_iterable = shift_ex_examples_rngs(ex_iterable=ex_iterable, value=worker_info.id)
+ self._state_dict = {
+ "examples_iterable": ex_iterable._init_state_dict(),
+ "epoch": self.epoch,
+ }
+ if self._starting_state_dict and self.epoch == self._starting_state_dict["epoch"]:
+ ex_iterable.load_state_dict(self._starting_state_dict["examples_iterable"])
+
+ if self._formatting and (ex_iterable.iter_arrow or self._formatting.is_table):
+ formatter = get_formatter(self._formatting.format_type, features=self.features)
+ for key, pa_table in ex_iterable.iter_arrow():
+ yield formatter.format_row(pa_table)
+ return
+ else:
+ for key, example in ex_iterable:
+ # no need to format thanks to FormattedExamplesIterable
+ yield example
+ logger.debug(
+ f"{_log_prefix}dataloader worker#{worker_info.id}, ': Finished iterating over {len(shards_indices)}/{ex_iterable.num_shards} shards."
+ )
+ else:
+ logger.debug(
+ f"{_log_prefix}dataloader worker#{worker_info.id}, ': Stopping... Number of dataset shards < num_workers ({ex_iterable.num_shards}<{worker_info.num_workers})."
+ )
+
+ def _is_main_process(self):
+ if self._distributed and self._distributed.rank > 0:
+ return False
+ if "torch" in sys.modules:
+ import torch.utils.data
+
+ worker_info = torch.utils.data.get_worker_info()
+ if worker_info is not None and worker_info.id > 0:
+ return False
+ return True
+
+ def _prepare_ex_iterable_for_iteration(
+ self, batch_size: int = 1, drop_last_batch: bool = False
+ ) -> _BaseExamplesIterable:
+ ex_iterable = self._ex_iterable
+
+ if self.epoch:
+ ex_iterable = ex_iterable.shuffle_data_sources(np.random.default_rng(self.epoch))
+ ex_iterable = shift_ex_examples_rngs(ex_iterable, self.epoch)
+
+ if self._distributed:
+ rank = self._distributed.rank
+ world_size = self._distributed.world_size
+ if ex_iterable.num_shards % world_size == 0:
+ if self._is_main_process():
+ num_shards_per_node = ex_iterable.num_shards // world_size
+ plural = "s" if num_shards_per_node > 1 else ""
+ logger.info(
+ f"Assigning {num_shards_per_node} shard{plural} (or data source{plural}) of the dataset to each node."
+ )
+ ex_iterable = ex_iterable.shard_data_sources(num_shards=world_size, index=rank, contiguous=False)
+ else:
+ if self._is_main_process():
+ logger.info(
+ f"Assigning 1 out of {world_size} examples of the dataset to each node. The others are skipped during the iteration."
+ )
+ logger.info(
+ f"It is more optimized to distribute the dataset shards (or data sources) across nodes. "
+ f"You can do that by using a dataset with number of shards that is a factor of world_size={world_size}. "
+ f"The current dataset has {ex_iterable.num_shards} which is not a factor of {world_size}"
+ )
+ ex_iterable = StepExamplesIterable(ex_iterable, step=world_size, offset=rank)
+
+ if ex_iterable.iter_arrow:
+ ex_iterable = RebatchedArrowExamplesIterable(
+ ex_iterable, batch_size=batch_size, drop_last_batch=drop_last_batch
+ )
+ elif self._formatting and self._formatting.is_table:
+ ex_iterable = RebatchedArrowExamplesIterable(
+ ex_iterable, batch_size=batch_size, drop_last_batch=drop_last_batch, force_convert_to_arrow=True
+ )
+
+ if self._formatting or (self.features and ex_iterable.features != self.features):
+ ex_iterable = FormattedExamplesIterable(
+ ex_iterable,
+ formatting=self._formatting,
+ features=self.features,
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ self._state_dict = {
+ "examples_iterable": ex_iterable._init_state_dict(),
+ "epoch": self.epoch,
+ }
+ if self._starting_state_dict and self.epoch == self._starting_state_dict["epoch"]:
+ ex_iterable.load_state_dict(self._starting_state_dict["examples_iterable"])
+ return ex_iterable
+
+ def __iter__(self):
+ if "torch" in sys.modules:
+ import torch.utils.data
+
+ worker_info = torch.utils.data.get_worker_info()
+ if isinstance(self, torch.utils.data.IterableDataset) and worker_info is not None:
+ # We're a torch.utils.data.IterableDataset in a PyTorch worker process
+ yield from self._iter_pytorch()
+ return
+
+ ex_iterable = self._prepare_ex_iterable_for_iteration()
+ if self._formatting and (ex_iterable.iter_arrow or self._formatting.is_table):
+ formatter = get_formatter(self._formatting.format_type, features=self.features)
+ for key, pa_table in ex_iterable.iter_arrow():
+ yield formatter.format_row(pa_table)
+ return
+
+ for key, example in ex_iterable:
+ # no need to format thanks to FormattedExamplesIterable
+ yield example
+
+ def iter(self, batch_size: int, drop_last_batch: bool = False):
+ """Iterate through the batches of size `batch_size`.
+
+ Args:
+ batch_size (:obj:`int`): size of each batch to yield.
+ drop_last_batch (:obj:`bool`, default `False`): Whether a last batch smaller than the batch_size should be
+ dropped
+ """
+
+ if self._formatting:
+ formatter = get_formatter(self._formatting.format_type, features=self.features)
+ format_dict = formatter.recursive_tensorize if isinstance(formatter, TensorFormatter) else None
+ else:
+ format_dict = None
+
+ ex_iterable = self._prepare_ex_iterable_for_iteration(batch_size=batch_size, drop_last_batch=drop_last_batch)
+ if self._formatting and (ex_iterable.iter_arrow or self._formatting.is_table):
+ for key, pa_table in ex_iterable.iter_arrow():
+ yield formatter.format_batch(pa_table)
+ return
+
+ iterator = iter(ex_iterable)
+ for key, example in iterator:
+ # If batched, first build the batch
+ examples = [example] + [example for key, example in islice(iterator, batch_size - 1)]
+ if drop_last_batch and len(examples) < batch_size: # ignore last batch
+ return
+ batch = _examples_to_batch(examples)
+ # we need to format here in case we need to stack tensors together
+ yield format_dict(batch) if format_dict else batch
+
+ def __getitem__(self, column_name: str) -> IterableColumn:
+ return IterableColumn(self, column_name)
+
+ @staticmethod
+ def from_generator(
+ generator: Callable,
+ features: Optional[Features] = None,
+ gen_kwargs: Optional[dict] = None,
+ split: NamedSplit = Split.TRAIN,
+ ) -> "IterableDataset":
+ """Create an Iterable Dataset from a generator.
+
+ Args:
+ generator (`Callable`):
+ A generator function that `yields` examples.
+ features (`Features`, *optional*):
+ Dataset features.
+ gen_kwargs(`dict`, *optional*):
+ Keyword arguments to be passed to the `generator` callable.
+ You can define a sharded iterable dataset by passing the list of shards in `gen_kwargs`.
+ This can be used to improve shuffling and when iterating over the dataset with multiple workers.
+ split ([`NamedSplit`], defaults to `Split.TRAIN`):
+ Split name to be assigned to the dataset.
+
+
+ Returns:
+ [`IterableDataset`]
+
+ Example:
+
+ ```py
+ >>> def gen():
+ ... yield {"text": "Good", "label": 0}
+ ... yield {"text": "Bad", "label": 1}
+ ...
+ >>> ds = IterableDataset.from_generator(gen)
+ ```
+
+ ```py
+ >>> def gen(shards):
+ ... for shard in shards:
+ ... with open(shard) as f:
+ ... for line in f:
+ ... yield {"line": line}
+ ...
+ >>> shards = [f"data{i}.txt" for i in range(32)]
+ >>> ds = IterableDataset.from_generator(gen, gen_kwargs={"shards": shards})
+ >>> ds = ds.shuffle(seed=42, buffer_size=10_000) # shuffles the shards order + uses a shuffle buffer
+ >>> from torch.utils.data import DataLoader
+ >>> dataloader = DataLoader(ds.with_format("torch"), num_workers=4) # give each worker a subset of 32/4=8 shards
+ ```
+ """
+ from .io.generator import GeneratorDatasetInputStream
+
+ return GeneratorDatasetInputStream(
+ generator=generator, features=features, gen_kwargs=gen_kwargs, streaming=True, split=split
+ ).read()
+
+ @staticmethod
+ def from_spark(
+ df: "pyspark.sql.DataFrame",
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ **kwargs,
+ ) -> "IterableDataset":
+ """Create an IterableDataset from Spark DataFrame. The dataset is streamed to the driver in batches.
+
+ Args:
+ df (`pyspark.sql.DataFrame`):
+ The DataFrame containing the desired data.
+ split (`NamedSplit`, *optional*):
+ Split name to be assigned to the dataset.
+ features (`Features`, *optional*):
+ Dataset features.
+
+ Returns:
+ [`IterableDataset`]
+
+ Example:
+
+ ```py
+ >>> df = spark.createDataFrame(
+ >>> data=[[1, "Elia"], [2, "Teo"], [3, "Fang"]],
+ >>> columns=["id", "name"],
+ >>> )
+ >>> ds = IterableDataset.from_spark(df)
+ ```
+ """
+ from .io.spark import SparkDatasetReader
+
+ if sys.platform == "win32":
+ raise OSError("IterableDataset.from_spark is not currently supported on Windows")
+
+ return SparkDatasetReader(
+ df,
+ split=split,
+ features=features,
+ streaming=True,
+ **kwargs,
+ ).read()
+
+ @staticmethod
+ def from_file(filename: str) -> "IterableDataset":
+ """Instantiate a IterableDataset from Arrow table at filename.
+
+ Args:
+ filename (`str`):
+ File name of the dataset.
+
+ Returns:
+ [`IterableDataset`]
+ """
+ pa_table_schema = read_schema_from_file(filename)
+ inferred_features = Features.from_arrow_schema(pa_table_schema)
+ ex_iterable = ArrowExamplesIterable(Dataset._generate_tables_from_cache_file, kwargs={"filename": filename})
+ return IterableDataset(ex_iterable=ex_iterable, info=DatasetInfo(features=inferred_features))
+
+ @classmethod
+ def from_pandas(
+ cls,
+ df: pd.DataFrame,
+ features: Optional[Features] = None,
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ preserve_index: Optional[bool] = None,
+ num_shards: Optional[int] = 1,
+ ) -> "IterableDataset":
+ """
+ Convert `pandas.DataFrame` to a `pyarrow.Table` to create an [`IterableDataset`].
+
+ The column types in the resulting Arrow Table are inferred from the dtypes of the `pandas.Series` in the
+ DataFrame. In the case of non-object Series, the NumPy dtype is translated to its Arrow equivalent. In the
+ case of `object`, we need to guess the datatype by looking at the Python objects in this Series.
+
+ Be aware that Series of the `object` dtype don't carry enough information to always lead to a meaningful Arrow
+ type. In the case that we cannot infer a type, e.g. because the DataFrame is of length 0 or the Series only
+ contains `None/nan` objects, the type is set to `null`. This behavior can be avoided by constructing explicit
+ features and passing it to this function.
+
+ Important: a dataset created with from_pandas() lives in memory.
+ This may change in the future, but in the meantime if you
+ want to reduce memory usage you should write it on disk
+ and reload using e.g. to_parquet / from_parquet.
+
+ Args:
+ df (`pandas.DataFrame`):
+ Dataframe that contains the dataset.
+ features ([`Features`], *optional*):
+ Dataset features.
+ info (`DatasetInfo`, *optional*):
+ Dataset information, like description, citation, etc.
+ split (`NamedSplit`, *optional*):
+ Name of the dataset split.
+ preserve_index (`bool`, *optional*):
+ Whether to store the index as an additional column in the resulting Dataset.
+ The default of `None` will store the index as a column, except for `RangeIndex` which is stored as metadata only.
+ Use `preserve_index=True` to force it to be stored as a column.
+ num_shards (`int`, default to `1`):
+ Number of shards to define when instantiating the iterable dataset. This is especially useful for big datasets to be able to shuffle properly,
+ and also to enable fast parallel loading using a PyTorch DataLoader or in distributed setups for example.
+
+ Returns:
+ [`IterableDataset`]
+
+ Example:
+
+ ```py
+ >>> ds = IterableDataset.from_pandas(df)
+ ```
+ """
+ return Dataset.from_pandas(
+ df,
+ features=features,
+ info=info,
+ split=split,
+ preserve_index=preserve_index,
+ ).to_iterable_dataset(num_shards=num_shards)
+
+ @classmethod
+ def from_polars(
+ cls,
+ df: Union["pl.DataFrame", "pl.LazyFrame"],
+ features: Optional[Features] = None,
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ ) -> "IterableDataset":
+ """
+ Create an IterableDataset from a polars DataFrame or LazyFrame.
+
+ Iterating over the dataset is mostly zero copy.
+ Under the hood, the dataset iterates over the polars DataFrame batches/slices.
+
+ Data types that do copy:
+ * CategoricalType
+
+ Args:
+ df (`polars.DataFrame`): DataFrame to convert to Arrow Table
+ features (`Features`, optional): Dataset features.
+ info (`DatasetInfo`, optional): Dataset information, like description, citation, etc.
+ split (`NamedSplit`, optional): Name of the dataset split.
+
+ Returns:
+ [`IterableDataset`]
+
+ Examples:
+ ```py
+ >>> ds = IterableDataset.from_polars(df)
+ ```
+ """
+ import polars as pl
+
+ if info is not None and features is not None and info.features != features:
+ raise ValueError(
+ f"Features specified in `features` and `info.features` can't be different:\n{features}\n{info.features}"
+ )
+ features = features if features is not None else info.features if info is not None else None
+ if features is not None:
+ features = _fix_for_backward_compatible_features(features)
+ if info is None:
+ info = DatasetInfo()
+ info.features = features or Features.from_arrow_schema(
+ (df.collect_schema() if isinstance(df, pl.LazyFrame) else df.schema).to_arrow()
+ )
+ return IterableDataset(
+ ArrowExamplesIterable(_generate_tables_from_polars, kwargs={"df": df}),
+ info=info,
+ split=split,
+ )
+
+ @classmethod
+ def from_dict(
+ cls,
+ mapping: dict,
+ features: Optional[Features] = None,
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ num_shards: Optional[int] = 1,
+ ) -> "IterableDataset":
+ """
+ Convert `dict` to a `pyarrow.Table` to create an [`IterableDataset`].
+
+ Important: a dataset created with from_dict() lives in memory.
+ This may change in the future, but in the meantime if you
+ want to reduce memory usage you should write it back on disk
+ and reload using e.g. to_parquet / from_parquet.
+
+ Args:
+ mapping (`Mapping`):
+ Mapping of strings to Arrays or Python lists.
+ features ([`Features`], *optional*):
+ Dataset features.
+ info (`DatasetInfo`, *optional*):
+ Dataset information, like description, citation, etc.
+ split (`NamedSplit`, *optional*):
+ Name of the dataset split.
+ num_shards (`int`, default to `1`):
+ Number of shards to define when instantiating the iterable dataset. This is especially useful for big datasets to be able to shuffle properly,
+ and also to enable fast parallel loading using a PyTorch DataLoader or in distributed setups for example.
+
+ Returns:
+ [`IterableDataset`]
+ """
+ return Dataset.from_dict(mapping, features=features, info=info, split=split).to_iterable_dataset(
+ num_shards=num_shards
+ )
+
+ @classmethod
+ def from_list(
+ cls,
+ mapping: list[dict],
+ features: Optional[Features] = None,
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ num_shards: Optional[int] = 1,
+ ) -> "IterableDataset":
+ """
+ Convert a list of dicts to a `pyarrow.Table` to create an [`IterableDataset`]`.
+
+ Note that the keys of the first entry will be used to determine the dataset columns,
+ regardless of what is passed to features.
+
+ Important: a dataset created with from_list() lives in memory.
+ This may change in the future, but in the meantime if you
+ want to reduce memory usage you should write it back on disk
+ and reload using e.g. from_parquet / to_parquet.
+
+ Args:
+ mapping (`List[dict]`): A list of mappings of strings to row values.
+ features (`Features`, optional): Dataset features.
+ info (`DatasetInfo`, optional): Dataset information, like description, citation, etc.
+ split (`NamedSplit`, optional): Name of the dataset split.
+ num_shards (`int`, default to `1`):
+ Number of shards to define when instantiating the iterable dataset. This is especially useful for big datasets to be able to shuffle properly,
+ and also to enable fast parallel loading using a PyTorch DataLoader or in distributed setups for example.
+
+ Returns:
+ [`IterableDataset`]
+ """
+ return Dataset.from_list(
+ mapping,
+ features=features,
+ info=info,
+ split=split,
+ ).to_iterable_dataset(num_shards=num_shards)
+
+ @staticmethod
+ def from_csv(
+ path_or_paths: Union[PathLike, list[PathLike]],
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ keep_in_memory: bool = False,
+ **kwargs,
+ ) -> "IterableDataset":
+ """Create an IterableDataset from CSV file(s).
+
+ Args:
+ path_or_paths (`path-like` or list of `path-like`):
+ Path(s) of the CSV file(s).
+ split ([`NamedSplit`], *optional*):
+ Split name to be assigned to the dataset.
+ features ([`Features`], *optional*):
+ Dataset features.
+ keep_in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+ **kwargs (additional keyword arguments):
+ Keyword arguments to be passed to [`pandas.read_csv`].
+
+ Returns:
+ [`IterableDataset`]
+
+ Example:
+
+ ```py
+ >>> ds = IterableDataset.from_csv('path/to/dataset.csv')
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.csv import CsvDatasetReader
+
+ return CsvDatasetReader(
+ path_or_paths,
+ split=split,
+ features=features,
+ keep_in_memory=keep_in_memory,
+ streaming=True,
+ **kwargs,
+ ).read()
+
+ @staticmethod
+ def from_json(
+ path_or_paths: Union[PathLike, list[PathLike]],
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ keep_in_memory: bool = False,
+ field: Optional[str] = None,
+ **kwargs,
+ ) -> "IterableDataset":
+ """Create an IterableDataset from JSON or JSON Lines file(s).
+
+ Args:
+ path_or_paths (`path-like` or list of `path-like`):
+ Path(s) of the JSON or JSON Lines file(s).
+ split ([`NamedSplit`], *optional*):
+ Split name to be assigned to the dataset.
+ features ([`Features`], *optional*):
+ Dataset features.
+ keep_in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+ field (`str`, *optional*):
+ Field name of the JSON file where the dataset is contained in.
+ **kwargs (additional keyword arguments):
+ Keyword arguments to be passed to [`JsonConfig`].
+
+ Returns:
+ [`IterableDataset`]
+
+ Example:
+
+ ```py
+ >>> ds = IterableDataset.from_json('path/to/dataset.json')
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.json import JsonDatasetReader
+
+ return JsonDatasetReader(
+ path_or_paths,
+ split=split,
+ features=features,
+ keep_in_memory=keep_in_memory,
+ field=field,
+ streaming=True,
+ **kwargs,
+ ).read()
+
+ @staticmethod
+ def from_parquet(
+ path_or_paths: Union[PathLike, list[PathLike]],
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ keep_in_memory: bool = False,
+ columns: Optional[list[str]] = None,
+ filters: Optional[Union[pds.Expression, list[tuple], list[list[tuple]]]] = None,
+ fragment_scan_options: Optional[pds.ParquetFragmentScanOptions] = None,
+ on_bad_files: Literal["error", "warn", "skip"] = "error",
+ **kwargs,
+ ) -> "IterableDataset":
+ """Create an IterableDataset from Parquet file(s).
+
+ Args:
+ path_or_paths (`path-like` or list of `path-like`):
+ Path(s) of the Parquet file(s).
+ split (`NamedSplit`, *optional*):
+ Split name to be assigned to the dataset.
+ features (`Features`, *optional*):
+ Dataset features.
+ keep_in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+ columns (`List[str]`, *optional*):
+ If not `None`, only these columns will be read from the file.
+ A column name may be a prefix of a nested field, e.g. 'a' will select
+ 'a.b', 'a.c', and 'a.d.e'.
+ filters (`Union[pyarrow.dataset.Expression, list[tuple], list[list[tuple]]]`, *optional*):
+ Return only the rows matching the filter.
+ If possible the predicate will be pushed down to exploit the partition information
+ or internal metadata found in the data source, e.g. Parquet statistics.
+ Otherwise filters the loaded RecordBatches before yielding them.
+ fragment_scan_options (`pyarrow.dataset.ParquetFragmentScanOptions`, *optional*)
+ Scan-specific options for Parquet fragments.
+ This is especially useful to configure buffering and caching.
+
+
+ on_bad_files (`Literal["error", "warn", "skip"]`, *optional*, defaults to "error")
+ Specify what to do upon encountering a bad file (a file that can't be read). Allowed values are :
+ * 'error', raise an Exception when a bad file is encountered.
+ * 'warn', raise a warning when a bad file is encountered and skip that file.
+ * 'skip', skip bad files without raising or warning when they are encountered.
+
+
+ **kwargs (additional keyword arguments):
+ Keyword arguments to be passed to [`ParquetConfig`].
+
+ Returns:
+ [`IterableDataset`]
+
+ Example:
+
+ ```py
+ >>> ds = IterableDataset.from_parquet('path/to/dataset.parquet')
+ ```
+
+ Load a subset of columns:
+
+ ```python
+ >>> ds = IterableDataset.from_parquet('path/to/dataset.parquet', columns=["col_0", "col_1"])
+ ```
+
+ Efficiently filter data, possibly skipping entire files or row groups:
+
+ ```python
+ >>> filters = [("col_0", "==", 0)]
+ >>> ds = IterableDataset.from_parquet(parquet_files_list, filters=filters)
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.parquet import ParquetDatasetReader
+
+ return ParquetDatasetReader(
+ path_or_paths,
+ split=split,
+ features=features,
+ keep_in_memory=keep_in_memory,
+ columns=columns,
+ streaming=True,
+ filters=filters,
+ fragment_scan_options=fragment_scan_options,
+ on_bad_files=on_bad_files,
+ **kwargs,
+ ).read()
+
+ @staticmethod
+ def from_text(
+ path_or_paths: Union[PathLike, list[PathLike]],
+ split: Optional[NamedSplit] = None,
+ features: Optional[Features] = None,
+ keep_in_memory: bool = False,
+ keep_linebreaks: bool = False,
+ sample_by: Literal["line", "paragraph", "document"] = "line",
+ **kwargs,
+ ) -> "IterableDataset":
+ """Create an IterableDataset from text file(s).
+
+ Args:
+ path_or_paths (`path-like` or list of `path-like`):
+ Path(s) of the text file(s).
+ split (`NamedSplit`, *optional*):
+ Split name to be assigned to the dataset.
+ features (`Features`, *optional*):
+ Dataset features.
+ keep_in_memory (`bool`, defaults to `False`):
+ Whether to copy the data in-memory.
+ keep_linebreaks: (`bool`, defaults to False):
+ Whether to keep line breaks.
+ sample_by (`Literal["line", "paragraph", "document"]`, defaults to "line"):
+ Whether to load data per line, praragraph or document.
+ By default one row in the dataset = one line.
+ **kwargs (additional keyword arguments):
+ Keyword arguments to be passed to [`TextConfig`].
+
+ Returns:
+ [`IterableDataset`]
+
+ Example:
+
+ ```py
+ >>> ds = IterableDataset.from_text('path/to/dataset.txt')
+ ```
+ """
+ # Dynamic import to avoid circular dependency
+ from .io.text import TextDatasetReader
+
+ return TextDatasetReader(
+ path_or_paths,
+ split=split,
+ features=features,
+ keep_in_memory=keep_in_memory,
+ streaming=True,
+ keep_linebreaks=keep_linebreaks,
+ sample_by=sample_by,
+ **kwargs,
+ ).read()
+
+ def with_format(
+ self,
+ type: Optional[str] = None,
+ ) -> "IterableDataset":
+ """
+ Return a dataset with the specified format.
+
+ Args:
+
+ type (`str`, *optional*):
+ Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`.
+ `None` means it returns python objects (default).
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> from transformers import AutoTokenizer
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation", streaming=True)
+ >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+ >>> ds = ds.map(lambda x: tokenizer(x['text'], truncation=True, padding=True), batched=True)
+ >>> ds = ds.with_format("torch")
+ >>> next(iter(ds))
+ {'text': 'compassionately explores the seemingly irreconcilable situation between conservative christian parents and their estranged gay and lesbian children .',
+ 'label': tensor(1),
+ 'input_ids': tensor([ 101, 18027, 16310, 16001, 1103, 9321, 178, 11604, 7235, 6617,
+ 1742, 2165, 2820, 1206, 6588, 22572, 12937, 1811, 2153, 1105,
+ 1147, 12890, 19587, 6463, 1105, 15026, 1482, 119, 102, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0]),
+ 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
+ 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])}
+ ```
+ """
+ type = get_format_type_from_alias(type)
+ # TODO(QL): add format_kwargs
+ # TODO(QL): add format_columns and return_all_columns
+ # TODO(QL): add pandas format
+ return IterableDataset(
+ ex_iterable=self._ex_iterable,
+ info=self._info.copy(),
+ split=self._split,
+ formatting=FormattingConfig(format_type=type),
+ distributed=deepcopy(self._distributed),
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def map(
+ self,
+ function: Optional[Callable] = None,
+ with_indices: bool = False,
+ input_columns: Optional[Union[str, list[str]]] = None,
+ batched: bool = False,
+ batch_size: Optional[int] = 1000,
+ drop_last_batch: bool = False,
+ remove_columns: Optional[Union[str, list[str]]] = None,
+ features: Optional[Features] = None,
+ fn_kwargs: Optional[dict] = None,
+ ) -> "IterableDataset":
+ """
+ Apply a function to all the examples in the iterable dataset (individually or in batches) and update them.
+ If your function returns a column that already exists, then it overwrites it.
+ The function is applied on-the-fly on the examples when iterating over the dataset.
+
+ You can specify whether the function should be batched or not with the `batched` parameter:
+
+ - If batched is `False`, then the function takes 1 example in and should return 1 example.
+ An example is a dictionary, e.g. `{"text": "Hello there !"}`.
+ - If batched is `True` and `batch_size` is 1, then the function takes a batch of 1 example as input and can return a batch with 1 or more examples.
+ A batch is a dictionary, e.g. a batch of 1 example is {"text": ["Hello there !"]}.
+ - If batched is `True` and `batch_size` is `n` > 1, then the function takes a batch of `n` examples as input and can return a batch with `n` examples, or with an arbitrary number of examples.
+ Note that the last batch may have less than `n` examples.
+ A batch is a dictionary, e.g. a batch of `n` examples is `{"text": ["Hello there !"] * n}`.
+
+ If the function is asynchronous, then `map` will run your function in parallel, with up to one thousand simulatenous calls.
+ It is recommended to use a `asyncio.Semaphore` in your function if you want to set a maximum number of operations that can run at the same time.
+
+ Args:
+ function (`Callable`, *optional*, defaults to `None`):
+ Function applied on-the-fly on the examples when you iterate on the dataset.
+ It must have one of the following signatures:
+
+ - `function(example: Dict[str, Any]) -> Dict[str, Any]` if `batched=False` and `with_indices=False`
+ - `function(example: Dict[str, Any], idx: int) -> Dict[str, Any]` if `batched=False` and `with_indices=True`
+ - `function(batch: Dict[str, List]) -> Dict[str, List]` if `batched=True` and `with_indices=False`
+ - `function(batch: Dict[str, List], indices: List[int]) -> Dict[str, List]` if `batched=True` and `with_indices=True`
+
+ For advanced usage, the function can also return a `pyarrow.Table`.
+ If the function is asynchronous, then `map` will run your function in parallel.
+ Moreover if your function returns nothing (`None`), then `map` will run your function and return the dataset unchanged.
+ If no function is provided, default to identity function: `lambda x: x`.
+ with_indices (`bool`, defaults to `False`):
+ Provide example indices to `function`. Note that in this case the signature of `function` should be `def function(example, idx[, rank]): ...`.
+ input_columns (`Optional[Union[str, List[str]]]`, defaults to `None`):
+ The columns to be passed into `function`
+ as positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument.
+ batched (`bool`, defaults to `False`):
+ Provide batch of examples to `function`.
+ batch_size (`int`, *optional*, defaults to `1000`):
+ Number of examples per batch provided to `function` if `batched=True`.
+ `batch_size <= 0` or `batch_size == None` then provide the full dataset as a single batch to `function`.
+ drop_last_batch (`bool`, defaults to `False`):
+ Whether a last batch smaller than the batch_size should be
+ dropped instead of being processed by the function.
+ remove_columns (`[List[str]]`, *optional*, defaults to `None`):
+ Remove a selection of columns while doing the mapping.
+ Columns will be removed before updating the examples with the output of `function`, i.e. if `function` is adding
+ columns with names in `remove_columns`, these columns will be kept.
+ features (`[Features]`, *optional*, defaults to `None`):
+ Feature types of the resulting dataset.
+ fn_kwargs (`Dict`, *optional*, default `None`):
+ Keyword arguments to be passed to `function`.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True)
+ >>> def add_prefix(example):
+ ... example["text"] = "Review: " + example["text"]
+ ... return example
+ >>> ds = ds.map(add_prefix)
+ >>> list(ds.take(3))
+ [{'label': 1,
+ 'text': 'Review: the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'},
+ {'label': 1,
+ 'text': 'Review: the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'},
+ {'label': 1, 'text': 'Review: effective but too-tepid biopic'}]
+ ```
+ """
+ return self._map(
+ function=function,
+ with_indices=with_indices,
+ input_columns=input_columns,
+ batched=batched,
+ batch_size=batch_size,
+ drop_last_batch=drop_last_batch,
+ remove_columns=remove_columns,
+ features=features,
+ fn_kwargs=fn_kwargs,
+ )
+
+ def _map(
+ self,
+ function: Optional[Callable] = None,
+ with_indices: bool = False,
+ input_columns: Optional[Union[str, list[str]]] = None,
+ batched: bool = False,
+ batch_size: Optional[int] = 1000,
+ drop_last_batch: bool = False,
+ remove_columns: Optional[Union[str, list[str]]] = None,
+ features: Optional[Features] = None,
+ fn_kwargs: Optional[dict] = None,
+ is_batch_accumulate_arrow_table_function: bool = False,
+ ) -> "IterableDataset":
+ if isinstance(input_columns, str):
+ input_columns = [input_columns]
+ if isinstance(remove_columns, str):
+ remove_columns = [remove_columns]
+ if function is None:
+ function = identity_func
+ if fn_kwargs is None:
+ fn_kwargs = {}
+ if features is not None:
+ features = _fix_for_backward_compatible_features(features)
+
+ ex_iterable = self._ex_iterable
+ # no need to apply features if ex_iterable is typed and if there was no cast_column()
+ input_features = (
+ None
+ if (ex_iterable.is_typed and (self._info.features is None or self._info.features == ex_iterable.features))
+ else self._info.features
+ )
+
+ if self._formatting and self._formatting.is_table:
+ # apply formatting before iter_arrow to keep map examples iterable happy
+ ex_iterable = FormattedExamplesIterable(
+ ex_iterable,
+ formatting=deepcopy(self._formatting),
+ features=input_features,
+ token_per_repo_id=self._token_per_repo_id,
+ )
+ ex_iterable = RebatchedArrowExamplesIterable(
+ ex_iterable,
+ batch_size=batch_size if batched else 1,
+ drop_last_batch=drop_last_batch,
+ force_convert_to_arrow=True,
+ )
+ else:
+ if self._ex_iterable.iter_arrow:
+ if self._formatting or input_features:
+ ex_iterable = RebatchedArrowExamplesIterable(
+ self._ex_iterable, batch_size=batch_size if batched else 1, drop_last_batch=drop_last_batch
+ )
+ if self._formatting or input_features:
+ # apply formatting after iter_arrow to avoid re-encoding the examples
+ ex_iterable = FormattedExamplesIterable(
+ ex_iterable,
+ formatting=deepcopy(self._formatting),
+ features=input_features,
+ token_per_repo_id=self._token_per_repo_id,
+ force_convert_to_python=True,
+ )
+
+ ex_iterable = MappedExamplesIterable(
+ ex_iterable,
+ function=function,
+ with_indices=with_indices,
+ input_columns=input_columns,
+ batched=batched,
+ batch_size=batch_size,
+ drop_last_batch=drop_last_batch,
+ remove_columns=remove_columns,
+ fn_kwargs=fn_kwargs,
+ formatting=self._formatting,
+ features=features,
+ is_batch_accumulate_arrow_table_function=is_batch_accumulate_arrow_table_function,
+ )
+ info = self.info.copy()
+ info.features = features
+ return IterableDataset(
+ ex_iterable=ex_iterable,
+ info=info,
+ split=self._split,
+ formatting=self._formatting,
+ distributed=deepcopy(self._distributed),
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def filter(
+ self,
+ function: Optional[Callable] = None,
+ with_indices=False,
+ input_columns: Optional[Union[str, list[str]]] = None,
+ batched: bool = False,
+ batch_size: Optional[int] = 1000,
+ fn_kwargs: Optional[dict] = None,
+ ) -> "IterableDataset":
+ """Apply a filter function to all the elements so that the dataset only includes examples according to the filter function.
+ The filtering is done on-the-fly when iterating over the dataset.
+
+ If the function is asynchronous, then `filter` will run your function in parallel, with up to one thousand simulatenous calls (configurable).
+ It is recommended to use a `asyncio.Semaphore` in your function if you want to set a maximum number of operations that can run at the same time.
+
+ Args:
+ function (`Callable`):
+ Callable with one of the following signatures:
+
+ - `function(example: Dict[str, Any]) -> bool` if `with_indices=False, batched=False`
+ - `function(example: Dict[str, Any], indices: int) -> bool` if `with_indices=True, batched=False`
+ - `function(example: Dict[str, List]) -> List[bool]` if `with_indices=False, batched=True`
+ - `function(example: Dict[str, List], indices: List[int]) -> List[bool]` if `with_indices=True, batched=True`
+
+ If the function is asynchronous, then `filter` will run your function in parallel.
+ If no function is provided, defaults to an always True function: `lambda x: True`.
+ with_indices (`bool`, defaults to `False`):
+ Provide example indices to `function`. Note that in this case the signature of `function` should be `def function(example, idx): ...`.
+ input_columns (`str` or `List[str]`, *optional*):
+ The columns to be passed into `function` as
+ positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument.
+ batched (`bool`, defaults to `False`):
+ Provide batch of examples to `function`.
+ batch_size (`int`, *optional*, default `1000`):
+ Number of examples per batch provided to `function` if `batched=True`.
+ fn_kwargs (`Dict`, *optional*, default `None`):
+ Keyword arguments to be passed to `function`.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True)
+ >>> ds = ds.filter(lambda x: x["label"] == 0)
+ >>> list(ds.take(3))
+ [{'label': 0, 'movie_review': 'simplistic , silly and tedious .'},
+ {'label': 0,
+ 'movie_review': "it's so laddish and juvenile , only teenage boys could possibly find it funny ."},
+ {'label': 0,
+ 'movie_review': 'exploitative and largely devoid of the depth or sophistication that would make watching such a graphic treatment of the crimes bearable .'}]
+ ```
+ """
+ if isinstance(input_columns, str):
+ input_columns = [input_columns]
+
+ # We need the examples to be decoded for certain feature types like Image or Audio,
+ # format and type before filtering
+ ex_iterable = self._ex_iterable
+ if self._info.features or self._formatting:
+ ex_iterable = FormattedExamplesIterable(
+ ex_iterable,
+ formatting=self._formatting,
+ features=ex_iterable.features if ex_iterable.is_typed else self._info.features,
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ ex_iterable = FilteredExamplesIterable(
+ ex_iterable,
+ function=function,
+ with_indices=with_indices,
+ input_columns=input_columns,
+ batched=batched,
+ batch_size=batch_size,
+ fn_kwargs=fn_kwargs,
+ formatting=self._formatting,
+ )
+ return IterableDataset(
+ ex_iterable=ex_iterable,
+ info=self._info,
+ split=self._split,
+ formatting=self._formatting,
+ distributed=deepcopy(self._distributed),
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def shuffle(
+ self,
+ seed=None,
+ generator: Optional[np.random.Generator] = None,
+ buffer_size: int = 1000,
+ max_buffer_input_shards: int = 10,
+ ) -> "IterableDataset":
+ """
+ Randomly shuffles the elements of this dataset.
+
+ This dataset fills a buffer with `buffer_size` elements, then randomly samples elements from this buffer,
+ replacing the selected elements with new elements. For perfect shuffling, a buffer size greater than or
+ equal to the full size of the dataset is required.
+
+ For instance, if your dataset contains 10,000 elements but `buffer_size` is set to 1000, then `shuffle` will
+ initially select a random element from only the first 1000 elements in the buffer. Once an element is
+ selected, its space in the buffer is replaced by the next (i.e. 1,001-st) element,
+ maintaining the 1000 element buffer.
+
+ If the dataset is made of several shards, it fills the buffer using up to `max_buffer_input_shards` shards
+ at a time and also does shuffle the order of the shards. This greatly improves the quality of the shuffling.
+
+ However if the order has been fixed by using [`~datasets.IterableDataset.skip`]
+ or [`~datasets.IterableDataset.take`] then the order of the shards is kept unchanged and only one shard at
+ a time is used to fill the buffer.
+
+ Args:
+ seed (`int`, *optional*, defaults to `None`):
+ Random seed that will be used to shuffle the dataset.
+ It is used to sample from the shuffle buffer and also to shuffle the data shards.
+ generator (`numpy.random.Generator`, *optional*):
+ Numpy random Generator to use to compute the permutation of the dataset rows.
+ If `generator=None` (default), uses `np.random.default_rng` (the default BitGenerator (PCG64) of NumPy).
+ buffer_size (`int`, defaults to `1000`):
+ Size of the buffer.
+ max_buffer_input_shards (`int`, defaults to `101000`):
+ Maximum number of shards to use to feed the buffer at a time.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True)
+ >>> list(ds.take(3))
+ [{'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .',
+ 'label': 1},
+ {'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .',
+ 'label': 1},
+ {'text': 'effective but too-tepid biopic', 'label': 1}]
+ >>> shuffled_ds = ds.shuffle(seed=42)
+ >>> list(shuffled_ds.take(3))
+ [{'text': "a sports movie with action that's exciting on the field and a story you care about off it .",
+ 'label': 1},
+ {'text': 'at its best , the good girl is a refreshingly adult take on adultery . . .',
+ 'label': 1},
+ {'text': "sam jones became a very lucky filmmaker the day wilco got dropped from their record label , proving that one man's ruin may be another's fortune .",
+ 'label': 1}]
+ >>> resharded_ds = ds.reshard() # useful to shard Parquet datasets by row group instead of by file, improving shuffling quality on dataset with one or few files.
+ >>> shuffled_resharded_ds = resharded_ds.shuffle(seed=42)
+ >>> list(shuffled_resharded_ds.take(3))
+ [{'text': 'this mistaken-identity picture is so film-culture referential that the final product is a ghost .',
+ 'label': 0},
+ {'text': 'woody allen used to ridicule movies like hollywood ending . now he makes them .',
+ 'label': 0},
+ {'text': "not only is undercover brother as funny , if not more so , than both austin powers films , but it's also one of the smarter , savvier spoofs to come along in some time .",
+ 'label': 1}]
+ ```
+ """
+ if generator is None:
+ generator = np.random.default_rng(seed)
+ else:
+ generator = deepcopy(generator)
+ ex_iterable = self._ex_iterable
+ try:
+ ex_iterable = ex_iterable.shuffle_data_sources(generator)
+ except DataSourcesShufflingDisallowed:
+ max_buffer_input_shards = 1
+ if ex_iterable.iter_arrow:
+ ex_iterable = RebatchedArrowExamplesIterable(ex_iterable, batch_size=1)
+ if max_buffer_input_shards > 1:
+ num_shards_to_interleave = min(ex_iterable.num_shards, max_buffer_input_shards)
+ ex_iterable = CyclingMultiSourcesExamplesIterable(
+ [
+ ex_iterable.shard_data_sources(num_shards=num_shards_to_interleave, index=index)
+ for index in range(num_shards_to_interleave)
+ ],
+ stopping_strategy="all_exhausted_without_replacement",
+ )
+ ex_iterable = BufferShuffledExamplesIterable(ex_iterable, buffer_size=buffer_size, generator=generator)
+ return IterableDataset(
+ ex_iterable,
+ info=self._info.copy(),
+ split=self._split,
+ formatting=self._formatting,
+ distributed=deepcopy(self._distributed),
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def set_epoch(self, epoch: int):
+ self._epoch += epoch - self._epoch # update torch value in shared memory in-place
+
+ def skip(self, n: int) -> "IterableDataset":
+ """
+ Create a new [`IterableDataset`] that skips the first `n` elements.
+
+ Args:
+ n (`int`):
+ Number of elements to skip.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True)
+ >>> list(ds.take(3))
+ [{'label': 1,
+ 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'},
+ {'label': 1,
+ 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'},
+ {'label': 1, 'text': 'effective but too-tepid biopic'}]
+ >>> ds = ds.skip(1)
+ >>> list(ds.take(3))
+ [{'label': 1,
+ 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'},
+ {'label': 1, 'text': 'effective but too-tepid biopic'},
+ {'label': 1,
+ 'text': 'if you sometimes like to go to the movies to have fun , wasabi is a good place to start .'}]
+ ```
+ """
+ ex_iterable = SkipExamplesIterable(
+ self._ex_iterable,
+ n,
+ split_when_sharding=self._distributed is None,
+ )
+ return IterableDataset(
+ ex_iterable=ex_iterable,
+ info=self._info.copy(),
+ split=self._split,
+ formatting=self._formatting,
+ distributed=deepcopy(self._distributed),
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def repeat(self, num_times: Optional[int]) -> "IterableDataset":
+ """
+ Create a new [`IterableDataset`] that repeats the underlying dataset `num_times` times.
+
+ N.B. The effect of calling shuffle after repeat depends significantly on buffer size.
+ With buffer_size 1, duplicate data is never seen in the same iteration, even after shuffling:
+ ds.repeat(n).shuffle(seed=42, buffer_size=1) is equivalent to ds.shuffle(seed=42, buffer_size=1).repeat(n),
+ and only shuffles shard orders within each iteration.
+ With buffer size >= (num samples in the dataset * num_times), we get full shuffling of the repeated data, i.e. we can observe duplicates in
+ the same iteration.
+
+ Args:
+ num_times (`int`) or (`None`):
+ Number of times to repeat the dataset. If `None`, the dataset will be repeated indefinitely.
+
+ Example:
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
+ >>> ds = ds.take(2).repeat(2)
+ >>> list(ds)
+ [{'label': 1,
+ 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'},
+ {'label': 1,
+ 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'},
+ {'label': 1, 'text': 'effective but too-tepid biopic'},
+ {'label': 1,
+ 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'},
+ {'label': 1,
+ 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'},
+ {'label': 1, 'text': 'effective but too-tepid biopic'}]
+ ```
+ """
+ return IterableDataset(
+ ex_iterable=RepeatExamplesIterable(self._ex_iterable, num_times=num_times),
+ info=self._info,
+ split=self._split,
+ formatting=self._formatting,
+ distributed=deepcopy(self._distributed),
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def take(self, n: int) -> "IterableDataset":
+ """
+ Create a new [`IterableDataset`] with only the first `n` elements.
+
+ Args:
+ n (`int`):
+ Number of elements to take.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True)
+ >>> small_ds = ds.take(2)
+ >>> list(small_ds)
+ [{'label': 1,
+ 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'},
+ {'label': 1,
+ 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}]
+ ```
+ """
+ ex_iterable = TakeExamplesIterable(
+ self._ex_iterable,
+ n,
+ split_when_sharding=self._distributed is None,
+ )
+ return IterableDataset(
+ ex_iterable=ex_iterable,
+ info=self._info.copy(),
+ split=self._split,
+ formatting=self._formatting,
+ distributed=deepcopy(self._distributed),
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def shard(
+ self,
+ num_shards: int,
+ index: int,
+ contiguous: bool = True,
+ ) -> "IterableDataset":
+ """Return the `index`-nth shard from dataset split into `num_shards` pieces.
+
+ This shards deterministically. `dataset.shard(n, i)` splits the dataset into contiguous chunks,
+ so it can be easily concatenated back together after processing. If `dataset.num_shards % n == l`, then the
+ first `l` datasets each have `(dataset.num_shards // n) + 1` shards, and the remaining datasets have `(dataset.num_shards // n)` shards.
+ `datasets.concatenate_datasets([dset.shard(n, i) for i in range(n)])` returns a dataset with the same order as the original.
+ In particular, `dataset.shard(dataset.num_shards, i)` returns a dataset with 1 shard.
+
+ Note: n should be less or equal to the number of shards in the dataset `dataset.num_shards`.
+
+ On the other hand, `dataset.shard(n, i, contiguous=False)` contains all the shards of the dataset whose index mod `n = i`.
+
+ Be sure to shard before using any randomizing operator (such as `shuffle`).
+ It is best if the shard operator is used early in the dataset pipeline.
+
+ Args:
+ num_shards (`int`):
+ How many shards to split the dataset into.
+ index (`int`):
+ Which shard to select and return.
+ contiguous: (`bool`, defaults to `True`):
+ Whether to select contiguous blocks of indices for shards.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("fancyzhx/amazon_polarity", split="train", streaming=True)
+ >>> ds
+ IterableDataset({
+ features: ['label', 'title', 'content'],
+ num_shards: 4
+ })
+ >>> ds.shard(num_shards=2, index=0)
+ IterableDataset({
+ features: ['label', 'title', 'content'],
+ num_shards: 2
+ })
+ ```
+ """
+ ex_iterable = self._ex_iterable.shard_data_sources(num_shards=num_shards, index=index, contiguous=contiguous)
+ return IterableDataset(
+ ex_iterable=ex_iterable,
+ info=self._info.copy(),
+ split=self._split,
+ formatting=self._formatting,
+ distributed=deepcopy(self._distributed),
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def reshard(self) -> "IterableDataset":
+ """Reshard the dataset if possible, i.e. split the current shards further into more shards.
+ This increases the number of shards and the resulting dataset has num_shards >= previous_num_shards.
+ Equality may happen if no shard can be split further.
+
+ The resharding mechanism depends on the dataset file format:
+
+ * Parquet: shard per row group instead of per file
+ * Other: not implemented yet (contributions are welcome !)
+
+ Be sure to reshard/shard before using any randomizing operator (such as `shuffle`).
+ It is best if the shard operator is used early in the dataset pipeline.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("fancyzhx/amazon_polarity", split="train", streaming=True)
+ >>> ds
+ IterableDataset({
+ features: ['label', 'title', 'content'],
+ num_shards: 4
+ })
+ >>> ds.reshard()
+ IterableDataset({
+ features: ['label', 'title', 'content'],
+ num_shards: 3600
+ })
+ ```
+ """
+ ex_iterable = self._ex_iterable.reshard_data_sources()
+ return IterableDataset(
+ ex_iterable=ex_iterable,
+ info=self._info.copy(),
+ split=self._split,
+ formatting=self._formatting,
+ distributed=deepcopy(self._distributed),
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def add_column(self, name: str, column: Union[list, np.array]) -> "IterableDataset":
+ """Add column to Dataset.
+
+ Args:
+ name (str): Column name.
+ column (list or np.array): Column data to be added.
+
+ Returns:
+ `IterableDataset`
+ """
+ return self.map(partial(add_column_fn, name=name, column=column), with_indices=True)
+
+ def rename_column(self, original_column_name: str, new_column_name: str) -> "IterableDataset":
+ """
+ Rename a column in the dataset, and move the features associated to the original column under the new column
+ name.
+
+ Args:
+ original_column_name (`str`):
+ Name of the column to rename.
+ new_column_name (`str`):
+ New name for the column.
+
+ Returns:
+ `IterableDataset`: A copy of the dataset with a renamed column.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True)
+ >>> next(iter(ds))
+ {'label': 1,
+ 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}
+ >>> ds = ds.rename_column("text", "movie_review")
+ >>> next(iter(ds))
+ {'label': 1,
+ 'movie_review': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}
+ ```
+ """
+ return self.rename_columns({original_column_name: new_column_name})
+
+ def rename_columns(self, column_mapping: dict[str, str]) -> "IterableDataset":
+ """
+ Rename several columns in the dataset, and move the features associated to the original columns under
+ the new column names.
+
+ Args:
+ column_mapping (`Dict[str, str]`): A mapping of columns to rename to their new names
+
+ Returns:
+ `IterableDataset`: A copy of the dataset with renamed columns
+ """
+
+ original_features = self._info.features.copy() if self._info.features else None
+ ds_iterable = self.map(
+ partial(_rename_columns_fn, column_mapping=column_mapping), remove_columns=list(column_mapping)
+ )
+ if original_features is not None:
+ ds_iterable._info.features = Features(
+ {
+ column_mapping[col] if col in column_mapping.keys() else col: feature
+ for col, feature in original_features.items()
+ }
+ )
+ return ds_iterable
+
+ def remove_columns(self, column_names: Union[str, list[str]]) -> "IterableDataset":
+ """
+ Remove one or several column(s) in the dataset and the features associated to them.
+ The removal is done on-the-fly on the examples when iterating over the dataset.
+
+
+ Args:
+ column_names (`Union[str, List[str]]`):
+ Name of the column(s) to remove.
+
+ Returns:
+ `IterableDataset`: A copy of the dataset object without the columns to remove.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True)
+ >>> next(iter(ds))
+ {'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .', 'label': 1}
+ >>> ds = ds.remove_columns("label")
+ >>> next(iter(ds))
+ {'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}
+ ```
+ """
+ original_features = self._info.features.copy() if self._info.features else None
+ ds_iterable = self.map(remove_columns=column_names)
+ if original_features is not None:
+ ds_iterable._info.features = original_features.copy()
+ for col, _ in original_features.items():
+ if col in column_names:
+ del ds_iterable._info.features[col]
+
+ return ds_iterable
+
+ def select_columns(self, column_names: Union[str, list[str]]) -> "IterableDataset":
+ """Select one or several column(s) in the dataset and the features
+ associated to them. The selection is done on-the-fly on the examples
+ when iterating over the dataset.
+
+
+ Args:
+ column_names (`Union[str, List[str]]`):
+ Name of the column(s) to select.
+
+ Returns:
+ `IterableDataset`: A copy of the dataset object with selected columns.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True)
+ >>> next(iter(ds))
+ {'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .', 'label': 1}
+ >>> ds = ds.select_columns("text")
+ >>> next(iter(ds))
+ {'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}
+ ```
+ """
+ if isinstance(column_names, str):
+ column_names = [column_names]
+
+ if self._info:
+ info = deepcopy(self._info)
+ if self._info.features is not None:
+ missing_columns = set(column_names) - set(self._info.features.keys())
+ if missing_columns:
+ raise ValueError(
+ f"Column name {list(missing_columns)} not in the "
+ "dataset. Columns in the dataset: "
+ f"{list(self._info.features.keys())}."
+ )
+ info.features = Features({c: info.features[c] for c in column_names})
+
+ ex_iterable = SelectColumnsIterable(self._ex_iterable, column_names)
+ return IterableDataset(
+ ex_iterable=ex_iterable,
+ info=info,
+ split=self._split,
+ formatting=self._formatting,
+ distributed=self._distributed,
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def cast_column(self, column: str, feature: FeatureType) -> "IterableDataset":
+ """Cast column to feature for decoding.
+
+ Args:
+ column (`str`):
+ Column name.
+ feature (`Feature`):
+ Target feature.
+
+ Returns:
+ `IterableDataset`
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset, Audio
+ >>> ds = load_dataset("PolyAI/minds14", name="en-US", split="train", streaming=True)
+ >>> ds.features
+ {'audio': Audio(sampling_rate=8000, mono=True, decode=True, id=None),
+ 'english_transcription': Value('string'),
+ 'intent_class': ClassLabel(num_classes=14, names=['abroad', 'address', 'app_error', 'atm_limit', 'balance', 'business_loan', 'card_issues', 'cash_deposit', 'direct_debit', 'freeze', 'high_value_payment', 'joint_account', 'latest_transactions', 'pay_bill']),
+ 'lang_id': ClassLabel(num_classes=14, names=['cs-CZ', 'de-DE', 'en-AU', 'en-GB', 'en-US', 'es-ES', 'fr-FR', 'it-IT', 'ko-KR', 'nl-NL', 'pl-PL', 'pt-PT', 'ru-RU', 'zh-CN']),
+ 'path': Value('string'),
+ 'transcription': Value('string')}
+ >>> ds = ds.cast_column("audio", Audio(sampling_rate=16000))
+ >>> ds.features
+ {'audio': Audio(sampling_rate=16000, mono=True, decode=True, id=None),
+ 'english_transcription': Value('string'),
+ 'intent_class': ClassLabel(num_classes=14, names=['abroad', 'address', 'app_error', 'atm_limit', 'balance', 'business_loan', 'card_issues', 'cash_deposit', 'direct_debit', 'freeze', 'high_value_payment', 'joint_account', 'latest_transactions', 'pay_bill']),
+ 'lang_id': ClassLabel(num_classes=14, names=['cs-CZ', 'de-DE', 'en-AU', 'en-GB', 'en-US', 'es-ES', 'fr-FR', 'it-IT', 'ko-KR', 'nl-NL', 'pl-PL', 'pt-PT', 'ru-RU', 'zh-CN']),
+ 'path': Value('string'),
+ 'transcription': Value('string')}
+ ```
+ """
+ feature = _fix_for_backward_compatible_features(feature)
+ info = self._info.copy()
+ info.features[column] = feature
+ return IterableDataset(
+ ex_iterable=self._ex_iterable,
+ info=info,
+ split=self._split,
+ formatting=self._formatting,
+ distributed=deepcopy(self._distributed),
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def cast(
+ self,
+ features: Features,
+ ) -> "IterableDataset":
+ """
+ Cast the dataset to a new set of features.
+
+ Args:
+ features ([`Features`]):
+ New features to cast the dataset to.
+ The name of the fields in the features must match the current column names.
+ The type of the data must also be convertible from one type to the other.
+ For non-trivial conversion, e.g. `string` <-> `ClassLabel` you should use [`~Dataset.map`] to update the Dataset.
+
+ Returns:
+ `IterableDataset`: A copy of the dataset with casted features.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset, ClassLabel, Value
+ >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True)
+ >>> ds.features
+ {'label': ClassLabel(names=['neg', 'pos']),
+ 'text': Value('string')}
+ >>> new_features = ds.features.copy()
+ >>> new_features["label"] = ClassLabel(names=["bad", "good"])
+ >>> new_features["text"] = Value("large_string")
+ >>> ds = ds.cast(new_features)
+ >>> ds.features
+ {'label': ClassLabel(names=['bad', 'good']),
+ 'text': Value('large_string')}
+ ```
+ """
+ features = _fix_for_backward_compatible_features(features)
+ info = self._info.copy()
+ info.features = features
+ return IterableDataset(
+ ex_iterable=self._ex_iterable,
+ info=info,
+ split=self._split,
+ formatting=self._formatting,
+ distributed=deepcopy(self._distributed),
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def decode(self, enable: bool = True, num_threads: int = 0) -> "IterableDataset":
+ """
+ Enable or disable the dataset features decoding for audio, image, video.
+
+ When enabled (default), media types are decoded:
+
+ * audio -> dict of "array" and "sampling_rate" and "path"
+ * image -> PIL.Image
+ * video -> torchcodec.decoders.VideoDecoder
+
+ You can enable multithreading using `num_threads`. This is especially useful to speed up remote
+ data streaming. However it can be slower than `num_threads=0` for local data on fast disks.
+
+ Disabling decoding is useful if you want to iterate on the paths or bytes of the media files
+ without actually decoding their content. To disable decoding you can use `.decode(False)`, which
+ is equivalent to calling `.cast()` or `.cast_column()` with all the Audio, Image and Video types
+ set to `decode=False`.
+
+ Args:
+ enable (`bool`, defaults to `True`):
+ Enable or disable features decoding.
+ num_threads (`int`, defaults to `0`):
+ Enable multithreading for features decoding.
+
+ Returns:
+ `IterableDataset`: A copy of the dataset with casted features.
+
+ Examples:
+
+ Disable decoding:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset("sshh12/planet-textures", split="train", streaming=True)
+ >>> next(iter(ds))
+ {'image': ,
+ 'text': 'A distant celestial object with an icy crust, displaying a light blue shade, covered with round pits and rugged terrains.'}
+ >>> ds = ds.decode(False)
+ >>> ds.features
+ {'image': Image(mode=None, decode=False, id=None),
+ 'text': Value('string')}
+ >>> next(iter(ds))
+ {
+ 'image': {
+ 'path': 'hf://datasets/sshh12/planet-textures@69dc4cef7a5c4b2cfe387727ec8ea73d4bff7302/train/textures/0000.png',
+ 'bytes': None
+ },
+ 'text': 'A distant celestial object with an icy crust, displaying a light blue shade, covered with round pits and rugged terrains.'
+ }
+ ```
+
+ Speed up streaming with multithreading:
+
+ ```py
+ >>> import os
+ >>> from datasets import load_dataset
+ >>> from tqdm import tqdm
+ >>> ds = load_dataset("sshh12/planet-textures", split="train", streaming=True)
+ >>> num_threads = min(32, (os.cpu_count() or 1) + 4)
+ >>> ds = ds.decode(num_threads=num_threads)
+ >>> for _ in tqdm(ds): # 20 times faster !
+ ... ...
+ ```
+ """
+ if not self.features:
+ raise ValueError(
+ "Features decoding is only available for datasets with known features, but features are Unknown. "
+ "Please set the datasets features with `ds = ds.cast(features)`."
+ )
+ ds = self
+
+ def set_decoding(decode: bool, feature):
+ if hasattr(feature, "decode"):
+ feature.decode = decode
+
+ if enable and num_threads > 0:
+ disabled_decoding_features = self.features.copy()
+ enabled_decoding_features = self.features.copy()
+
+ _visit(disabled_decoding_features, partial(set_decoding, False))
+ _visit(enabled_decoding_features, partial(set_decoding, True))
+ ds = ds.cast(disabled_decoding_features)
+ pool = multiprocessing.pool.ThreadPool(num_threads)
+ func = partial(_apply_async, pool, enabled_decoding_features.decode_example)
+ ds = ds.map(func, features=enabled_decoding_features)
+ assert isinstance(ds._ex_iterable, MappedExamplesIterable)
+ ds._ex_iterable.max_num_running_async_map_functions_in_parallel = 2 * num_threads
+ else:
+ features = ds.features.copy()
+ _visit(features, partial(set_decoding, enable))
+ ds = ds.cast(features)
+ return ds
+
+ def _step(self, step: int, offset: int) -> "IterableDataset":
+ ex_iterable = StepExamplesIterable(self._ex_iterable, step=step, offset=offset)
+ return IterableDataset(
+ ex_iterable=ex_iterable,
+ info=self._info.copy(),
+ split=self._split,
+ formatting=self._formatting,
+ distributed=deepcopy(self._distributed),
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def _resolve_features(self):
+ if self.features is not None:
+ return self
+ elif self._ex_iterable.is_typed:
+ features = self._ex_iterable.features
+ else:
+ features = _infer_features_from_batch(self.with_format(None)._head())
+ info = self.info.copy()
+ info.features = features
+ return IterableDataset(
+ ex_iterable=self._ex_iterable,
+ info=info,
+ split=self._split,
+ formatting=self._formatting,
+ distributed=deepcopy(self._distributed),
+ token_per_repo_id=self._token_per_repo_id,
+ )
+
+ def batch(
+ self,
+ batch_size: Optional[int] = None,
+ by_column: Optional[Union[str, list[str]]] = None,
+ drop_last_batch: bool = False,
+ ) -> "IterableDataset":
+ """
+ Group samples from the dataset into batches.
+
+ Args:
+ batch_size (`int`, optional):
+ The number of samples in each batch.
+ by_column (`Union[str, list[str]`, optional):
+ The column used to batch examples together.
+ Successive examples with the same value for that column are in grouped the same batch.
+ This can also be a list of columns if you want to batch by multiple columns.
+ If batching by column, the batch_size is only used to control the size of the batches
+ to group together or slice during acculumation.
+
+
+ drop_last_batch (`bool`, defaults to `False`):
+ Whether to drop the last incomplete batch.
+
+ Example:
+ ```py
+ >>> ds = load_dataset("some_dataset", streaming=True)
+ >>> batched_ds = ds.batch(batch_size=32)
+ ```
+ """
+ if batch_size is None and by_column is None:
+ raise ValueError("IterableDataset.batch() misses `batch_size` or `by_column` argument.")
+ if self.features:
+ features = Features({col: List(feature) for col, feature in self.features.items()})
+ else:
+ features = None
+ if by_column is not None:
+ columns = [by_column] if isinstance(by_column, str) else by_column
+ ds = (
+ self.with_format("arrow")
+ ._map(
+ partial(_batch_accumulate_arrow_table_by_columns, columns=columns),
+ with_indices=True,
+ batched=True,
+ batch_size=batch_size,
+ drop_last_batch=drop_last_batch,
+ features=features,
+ is_batch_accumulate_arrow_table_function=True,
+ )
+ .with_format(self._formatting.format_type if self._formatting else None)
+ )
+ return ds
+ if self._formatting and self._formatting.is_table:
+ return (
+ self.with_format("arrow")
+ .map(
+ _batch_arrow_table,
+ batched=True,
+ batch_size=batch_size,
+ drop_last_batch=drop_last_batch,
+ features=features,
+ )
+ .with_format(self._formatting.format_type)
+ )
+ return self.map(
+ _batch_fn, batched=True, batch_size=batch_size, drop_last_batch=drop_last_batch, features=features
+ )
+
+ def to_dict(self, batch_size: Optional[int] = None, batched: bool = False) -> Union[dict, Iterator[dict]]:
+ """Returns the dataset as a Python dict. Can also return a generator for large datasets.
+
+ Args:
+ batch_size (`int`, *optional*): The size (number of rows) of the batches if `batched` is `True`.
+ Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+
+ Returns:
+ `dict` or `Iterator[dict]`
+
+ Example:
+
+ ```py
+ >>> ds.to_dict()
+ ```
+ """
+ if batched:
+ for table in self.with_format("arrow").iter(batch_size=batch_size):
+ yield Dataset(table, fingerprint="unset").to_dict()
+ else:
+ table = pa.concat_tables(list(self.with_format("arrow").iter(batch_size=1000)))
+ return Dataset(table, fingerprint="unset").to_dict()
+
+ def to_list(self) -> list:
+ """Returns the dataset as a Python list.
+
+ Returns:
+ `list`
+
+ Example:
+
+ ```py
+ >>> ds.to_list()
+ ```
+ """
+ table = pa.concat_tables(list(self.with_format("arrow").iter(batch_size=1000)))
+ return Dataset(table, fingerprint="unset").to_list()
+
+ def to_pandas(
+ self, batch_size: Optional[int] = None, batched: bool = False
+ ) -> Union[pd.DataFrame, Iterator[pd.DataFrame]]:
+ """Returns the dataset as a `pandas.DataFrame`. Can also return a generator for large datasets.
+
+ Args:
+ batch_size (`int`, *optional*):
+ The size (number of rows) of the batches if `batched` is `True`.
+ Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+ batched (`bool`):
+ Set to `True` to return a generator that yields the dataset as batches
+ of `batch_size` rows. Defaults to `False` (returns the whole datasets once).
+
+ Returns:
+ `pandas.DataFrame` or `Iterator[pandas.DataFrame]`
+
+ Example:
+
+ ```py
+ >>> ds.to_pandas()
+ ```
+ """
+ info = DatasetInfo(features=self.features.copy()) if self.features is not None else None
+
+ def maybe_cast_to_declared_features(table: pa.Table):
+ if self.features is not None and table.schema != self.features.arrow_schema:
+ return cast_table_to_features(table, self.features)
+ return table
+
+ if batched:
+ return (
+ Dataset(maybe_cast_to_declared_features(table), info=info, fingerprint="unset").to_pandas()
+ for table in self.with_format("arrow").iter(batch_size=batch_size)
+ )
+ else:
+ table = pa.concat_tables(
+ [maybe_cast_to_declared_features(table) for table in self.with_format("arrow").iter(batch_size=1000)]
+ )
+ return Dataset(table, info=info, fingerprint="unset").to_pandas()
+
+ def to_polars(
+ self,
+ batch_size: Optional[int] = None,
+ batched: bool = False,
+ schema_overrides: Optional[dict] = None,
+ rechunk: bool = True,
+ ) -> Union["pl.DataFrame", Iterator["pl.DataFrame"]]:
+ """Returns the dataset as a `polars.DataFrame`. Can also return a generator for large datasets.
+
+ Args:
+ batch_size (`int`, *optional*):
+ The size (number of rows) of the batches if `batched` is `True`.
+ Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+ batched (`bool`):
+ Set to `True` to return a generator that yields the dataset as batches
+ of `batch_size` rows. Defaults to `False` (returns the whole datasets once).
+ schema_overrides (`dict`, *optional*):
+ Support type specification or override of one or more columns; note that
+ any dtypes inferred from the schema param will be overridden.
+ rechunk (`bool`):
+ Make sure that all data is in contiguous memory. Defaults to `True`.
+ Returns:
+ `polars.DataFrame` or `Iterator[polars.DataFrame]`
+
+ Example:
+
+ ```py
+ >>> ds.to_polars()
+ ```
+ """
+ if batched:
+ for table in self.with_format("arrow").iter(batch_size=batch_size):
+ yield Dataset(table, fingerprint="unset").to_polars(schema_overrides=schema_overrides, rechunk=rechunk)
+ else:
+ table = pa.concat_tables(list(self.with_format("arrow").iter(batch_size=1000)))
+ return Dataset(table, fingerprint="unset").to_polars(schema_overrides=schema_overrides, rechunk=rechunk)
+
+ def to_csv(
+ self,
+ path_or_buf: Union[PathLike, BinaryIO],
+ batch_size: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ **to_csv_kwargs,
+ ) -> int:
+ """Exports the dataset to csv.
+
+ This iterates on the dataset and loads it completely in memory before writing it.
+
+ Args:
+ path_or_buf (`PathLike` or `FileOrBuffer`):
+ Either a path to a file (e.g. `file.csv`), a remote URI (e.g. `hf://datasets/username/my_dataset_name/data.csv`),
+ or a BinaryIO, where the dataset will be saved to in the specified format.
+ batch_size (`int`, *optional*):
+ Size of the batch to load in memory and write at once.
+ Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+ **to_csv_kwargs (additional keyword arguments):
+ Parameters to pass to pandas's [`pandas.DataFrame.to_csv`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_csv.html).
+ The parameter `index` defaults to `False` if not specified.
+ If you would like to write the index, pass `index=True` and also set a name for the index column by
+ passing `index_label`.
+
+ Returns:
+ `int`: The number of characters or bytes written.
+
+ Example:
+
+ ```py
+ >>> ds.to_csv("path/to/dataset/directory")
+ ```
+ """
+ table = pa.concat_tables(list(self.with_format("arrow").iter(batch_size=1000)))
+ return Dataset(table, fingerprint="unset").to_csv(
+ path_or_buf,
+ batch_size=batch_size,
+ storage_options=storage_options,
+ **to_csv_kwargs,
+ )
+
+ def to_json(
+ self,
+ path_or_buf: Union[PathLike, BinaryIO],
+ batch_size: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ **to_json_kwargs,
+ ) -> int:
+ """Export the dataset to JSON Lines or JSON.
+
+ This iterates on the dataset and loads it completely in memory before writing it.
+
+ The default output format is [JSON Lines](https://jsonlines.org/).
+ To export to [JSON](https://www.json.org), pass `lines=False` argument and the desired `orient`.
+
+ Args:
+ path_or_buf (`PathLike` or `FileOrBuffer`):
+ Either a path to a file (e.g. `file.json`), a remote URI (e.g. `hf://datasets/username/my_dataset_name/data.json`),
+ or a BinaryIO, where the dataset will be saved to in the specified format.
+ batch_size (`int`, *optional*):
+ Size of the batch to load in memory and write at once.
+ Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+ **to_json_kwargs (additional keyword arguments):
+ Parameters to pass to pandas's [`pandas.DataFrame.to_json`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_json.html).
+ Default arguments are `lines=True` and `orient="records".
+ The parameter `index` defaults to `False` if `orient` is `"split"` or `"table"`.
+ If you would like to write the index, pass `index=True`.
+
+ Returns:
+ `int`: The number of characters or bytes written.
+
+ Example:
+
+ ```py
+ >>> ds.to_json("path/to/dataset/directory/filename.jsonl")
+ ```
+
+ ```py
+ >>> num_shards = dataset.num_shards
+ >>> for index in range(num_shards):
+ ... shard = dataset.shard(index, num_shards)
+ ... shard.to_json(f"path/of/my/dataset/data-{index:05d}.jsonl")
+ ```
+
+ """
+ table = pa.concat_tables(list(self.with_format("arrow").iter(batch_size=1000)))
+ return Dataset(table, fingerprint="unset").to_json(
+ path_or_buf,
+ batch_size=batch_size,
+ storage_options=storage_options,
+ **to_json_kwargs,
+ )
+
+ def to_sql(
+ self,
+ name: str,
+ con: Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"],
+ batch_size: Optional[int] = None,
+ **sql_writer_kwargs,
+ ) -> int:
+ """Exports the dataset to a SQL database.
+
+ Args:
+ name (`str`):
+ Name of SQL table.
+ con (`str` or `sqlite3.Connection` or `sqlalchemy.engine.Connection` or `sqlalchemy.engine.Connection`):
+ A [URI string](https://docs.sqlalchemy.org/en/13/core/engines.html#database-urls) or a SQLite3/SQLAlchemy connection object used to write to a database.
+ batch_size (`int`, *optional*):
+ Size of the batch to load in memory and write at once.
+ Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+ **sql_writer_kwargs (additional keyword arguments):
+ Parameters to pass to pandas's [`pandas.DataFrame.to_sql`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_sql.html).
+ The parameter `index` defaults to `False` if not specified.
+ If you would like to write the index, pass `index=True` and also set a name for the index column by
+ passing `index_label`.
+
+
+ Returns:
+ `int`: The number of records written.
+
+ Example:
+
+ ```py
+ >>> # con provided as a connection URI string
+ >>> ds.to_sql("data", "sqlite:///my_own_db.sql")
+ >>> # con provided as a sqlite3 connection object
+ >>> import sqlite3
+ >>> con = sqlite3.connect("my_own_db.sql")
+ >>> with con:
+ ... ds.to_sql("data", con)
+ ```
+ """
+ table = pa.concat_tables(list(self.with_format("arrow").iter(batch_size=1000)))
+ return Dataset(table, fingerprint="unset").to_sql(name, con, batch_size=batch_size, **sql_writer_kwargs)
+
+ def to_parquet(
+ self,
+ path_or_buf: Union[PathLike, BinaryIO],
+ batch_size: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ **parquet_writer_kwargs,
+ ) -> int:
+ """Exports the dataset to parquet
+
+ Args:
+ path_or_buf (`PathLike` or `FileOrBuffer`):
+ Either a path to a file (e.g. `file.parquet`), a remote URI (e.g. `hf://datasets/username/my_dataset_name/data.parquet`),
+ or a BinaryIO, where the dataset will be saved to in the specified format.
+ batch_size (`int`, *optional*):
+ Size of the batch to load in memory and write at once.
+ Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+
+
+ **parquet_writer_kwargs (additional keyword arguments):
+ Parameters to pass to PyArrow's `pyarrow.parquet.ParquetWriter`.
+
+ Returns:
+ `int`: The number of characters or bytes written.
+
+ Example:
+
+ ```py
+ >>> ds.to_parquet("path/to/dataset/directory")
+ ```
+
+ ```py
+ >>> num_shards = dataset.num_shards
+ >>> for index in range(num_shards):
+ ... shard = dataset.shard(index, num_shards)
+ ... shard.to_parquet(f"path/of/my/dataset/data-{index:05d}.parquet")
+ ```
+
+ """
+ from .arrow_writer import get_arrow_writer_batch_size_from_features
+
+ batch_size = get_arrow_writer_batch_size_from_features(self.features) or config.DEFAULT_MAX_BATCH_SIZE
+ table = pa.concat_tables(list(self.with_format("arrow").iter(batch_size=batch_size)))
+ return Dataset(table, fingerprint="unset").to_parquet(
+ path_or_buf, storage_options=storage_options, **parquet_writer_kwargs
+ )
+
+ def _push_parquet_shards_to_hub_single(
+ self,
+ job_id: int,
+ num_jobs: int,
+ resolved_output_path: HfFileSystemResolvedPath,
+ data_dir: str,
+ split: str,
+ token: Optional[str],
+ create_pr: Optional[bool],
+ # max_shard_size: Optional[Union[int, str]] = None, # TODO(QL): add arg
+ num_shards: int,
+ embed_external_files: bool,
+ ) -> Iterable[tuple[list[CommitOperationAdd], list[str], int, int]]:
+ """Pushes the dataset shards as Parquet files to the hub.
+
+ Returns:
+ additions (`List[CommitOperation]`): list of the `CommitOperationAdd` of the uploaded shards
+ new_parquet_paths (`List[str]`): list of the paths of the uploaded parquet files
+ features (`Features`): features of the uploaded dataset
+ dataset_nbytes (`int`): approximate size in bytes of the uploaded dataset after uncompression
+ num_examples (`int`): number of examples of th euploaded shards
+ """
+
+ div = num_shards // num_jobs
+ mod = num_shards % num_jobs
+ start = div * job_id + min(job_id, mod)
+ end = start + div + (1 if job_id < mod else 0)
+
+ index_shards = (
+ (start + i, self.shard(num_shards=end - start, index=i, contiguous=True)) for i in range(end - start)
+ )
+
+ api = HfApi(endpoint=config.HF_ENDPOINT, token=token, library_name="datasets", library_version=__version__)
+
+ dataset_nbytes = 0
+ num_examples = 0
+ additions: list[CommitOperationAdd] = []
+ new_parquet_paths: list[str] = []
+ features = self.features
+ for index, shard in index_shards:
+ if embed_external_files:
+ from .arrow_writer import get_arrow_writer_batch_size_from_features
+
+ shard = shard.with_format("arrow")
+ shard = shard.map(
+ partial(embed_table_storage, token_per_repo_id=self._token_per_repo_id),
+ batched=True,
+ batch_size=get_arrow_writer_batch_size_from_features(shard.features),
+ )
+ shard_path_in_repo = f"{data_dir}/{split}-{index:05d}-of-{num_shards:05d}.parquet"
+ tmp_file = tempfile.NamedTemporaryFile(suffix=".parquet", delete=False)
+ try:
+ shard.to_parquet(tmp_file)
+ tmp_file.close()
+ parquet_metadata = pq.read_metadata(tmp_file.name)
+ if features is None:
+ features = Features.from_arrow_schema(parquet_metadata.schema.to_arrow_schema())
+ num_examples += parquet_metadata.num_rows
+ dataset_nbytes += sum(
+ parquet_metadata.row_group(i).total_byte_size for i in range(parquet_metadata.num_row_groups)
+ )
+ new_parquet_paths.append(shard_path_in_repo)
+ if (
+ isinstance(resolved_output_path, HfFileSystemResolvedRepositoryPath)
+ and not resolved_output_path.path_in_repo
+ ):
+ shard_addition = CommitOperationAdd(path_in_repo=shard_path_in_repo, path_or_fileobj=tmp_file.name)
+ api.preupload_lfs_files(
+ repo_id=resolved_output_path.repo_id,
+ additions=[shard_addition],
+ repo_type=resolved_output_path.repo_type,
+ revision=resolved_output_path.revision,
+ create_pr=create_pr,
+ )
+ additions.append(shard_addition)
+ elif isinstance(resolved_output_path, HfFileSystemResolvedBucketPath):
+ if resolved_output_path.path:
+ shard_path_in_repo = resolved_output_path.path + "/" + shard_path_in_repo
+ api.batch_bucket_files(
+ bucket_id=resolved_output_path.bucket_id, add=[(tmp_file.name, shard_path_in_repo)]
+ )
+ else:
+ raise NotImplementedError(f"Bad HF path: {resolved_output_path}")
+ except (Exception, KeyboardInterrupt):
+ tmp_file.close()
+ Path(tmp_file.name).unlink()
+ raise
+ tmp_file.close()
+ Path(tmp_file.name).unlink()
+ yield job_id, False, 1
+
+ yield job_id, True, (additions, new_parquet_paths, features, dataset_nbytes, num_examples)
+
+ def _push_parquet_shards_to_hub(
+ self,
+ resolved_output_path: HfFileSystemResolvedPath,
+ data_dir: str,
+ split: str,
+ token: Optional[str],
+ create_pr: Optional[bool],
+ max_shard_size: Optional[Union[int, str]],
+ num_shards: Optional[int],
+ embed_external_files: bool,
+ num_proc: Optional[int],
+ ) -> tuple[list[CommitOperationAdd], list[str], Features, SplitInfo, int]:
+ """Pushes the dataset shards as Parquet files to the hub.
+
+ Returns:
+ additions (`List[CommitOperation]`): list of the `CommitOperationAdd` of the uploaded shards
+ new_parquet_paths (`List[str]`): list of paths of the new files uploaded to the output path,
+ relative to output path
+ features (`features`): features of the uploaded dataset
+ split_info (`int`): info of the uploaded split, including the approximate size in bytes of
+ the uploaded dataset after uncompression
+ uploaded_size (`int`): number of uploaded bytes to the repository or bucket
+ """
+
+ # Find decodable columns, because if there are any, we need to:
+ # embed the bytes from the files in the shards
+ decodable_columns = (
+ [k for k, v in self._info.features.items() if require_decoding(v, ignore_decode_attribute=True)]
+ if embed_external_files
+ else []
+ )
+ embed_external_files = embed_external_files and bool(decodable_columns)
+
+ if num_shards is None:
+ if max_shard_size is None:
+ num_shards = self.num_shards
+ else:
+ max_shard_size = convert_file_size_to_int(max_shard_size or config.MAX_SHARD_SIZE)
+ estimated_nbytes = 0
+ for pa_table in self.with_format("arrow").iter(batch_size=config.DEFAULT_MAX_BATCH_SIZE):
+ estimated_nbytes += pa_table.nbytes
+ num_shards = int(estimated_nbytes / max_shard_size) + 1
+ num_shards = max(num_shards, num_proc or 1)
+
+ additions: list[CommitOperationAdd] = []
+ new_parquet_paths: list[str] = []
+ uploaded_size = 0
+ dataset_nbytes = 0
+ num_examples = 0
+ features = self.features
+
+ num_jobs = num_proc or 1
+ if num_shards <= 1:
+ logger.warning(
+ f"Setting num_proc from {num_jobs} back to 1 for the {split} split to disable multiprocessing as it only contains one shard."
+ )
+ num_proc = None
+ num_jobs = 1
+ elif num_shards < num_jobs:
+ logger.warning(
+ f"Setting num_proc from {num_jobs} to {num_shards} for the {split} split as it only contains {num_shards} shards."
+ )
+ num_proc = num_shards
+ num_jobs = num_shards
+ kwargs_iterable = [
+ {
+ "self": self.shard(num_shards=num_jobs, index=job_id, contiguous=True),
+ "job_id": job_id,
+ "num_jobs": num_jobs,
+ "resolved_output_path": resolved_output_path,
+ "data_dir": data_dir,
+ "split": split,
+ "token": token,
+ "create_pr": create_pr,
+ "num_shards": num_shards,
+ "embed_external_files": embed_external_files,
+ }
+ for job_id in range(num_jobs)
+ ]
+ desc = "Uploading the dataset shards"
+ desc += f" (num_proc={num_proc})" if num_proc is not None and num_proc >= 1 else ""
+ pbar = hf_tqdm(
+ unit=" shards",
+ total=num_shards,
+ desc=desc,
+ )
+ with (
+ contextlib.nullcontext()
+ if num_proc is None or num_proc < 1
+ else mp.get_context("spawn").Pool(num_proc) as pool
+ ):
+ update_stream = (
+ IterableDataset._push_parquet_shards_to_hub_single(**kwargs_iterable[0])
+ if pool is None
+ else iflatmap_unordered(
+ pool,
+ IterableDataset._push_parquet_shards_to_hub_single,
+ kwargs_iterable=kwargs_iterable,
+ )
+ )
+ for job_id, done, content in update_stream:
+ if not done:
+ pbar.update(content)
+ else:
+ job_additions, job_new_parquet_paths, job_features, job_uploaded_size, job_num_examples = content
+ additions += job_additions
+ new_parquet_paths += job_new_parquet_paths
+ uploaded_size += job_uploaded_size
+ num_examples += job_num_examples
+ features = job_features
+ if pool is not None:
+ pool.close()
+ pool.join()
+
+ uploaded_size = sum(addition.upload_info.size for addition in additions)
+ split_info = SplitInfo(split, num_bytes=dataset_nbytes, num_examples=num_examples)
+ return additions, new_parquet_paths, features, split_info, uploaded_size
+
+ def push_to_hub(
+ self,
+ repo_id: str,
+ config_name: str = "default",
+ set_default: Optional[bool] = None,
+ split: Optional[str] = None,
+ data_dir: Optional[str] = None,
+ commit_message: Optional[str] = None,
+ commit_description: Optional[str] = None,
+ private: Optional[bool] = None,
+ token: Optional[str] = None,
+ revision: Optional[str] = None,
+ create_pr: Optional[bool] = False,
+ max_shard_size: Optional[Union[int, str]] = None,
+ num_shards: Optional[int] = None,
+ embed_external_files: bool = True,
+ num_proc: Optional[int] = None,
+ ) -> CommitInfo:
+ """Pushes the dataset to the hub as a Parquet dataset.
+ The dataset is pushed using HTTP requests and does not need to have neither git or git-lfs installed.
+
+ The resulting Parquet files are self-contained by default. If your dataset contains [`Image`], [`Audio`] or [`Video`]
+ data, the Parquet files will store the bytes of your images or audio files.
+ You can disable this by setting `embed_external_files` to `False`.
+
+ Args:
+ repo_id (`str`):
+ The ID of the repository to push to in the following format: `/` or
+ `/`. Also accepts ``, which will default to the namespace
+ of the logged-in user.
+
+ It could also be a location inside a bucket, e.g. `buckets///...`
+ config_name (`str`, defaults to "default"):
+ The configuration name (or subset) of a dataset. Defaults to "default".
+ set_default (`bool`, *optional*):
+ Whether to set this configuration as the default one. Otherwise, the default configuration is the one
+ named "default".
+ split (`str`, *optional*):
+ The name of the split that will be given to that dataset. Defaults to `self.split`.
+ data_dir (`str`, *optional*):
+ Directory name that will contain the uploaded data files. Defaults to the `config_name` if different
+ from "default", else "data".
+ commit_message (`str`, *optional*):
+ Message to commit while pushing. Will default to `"Upload dataset"`.
+ commit_description (`str`, *optional*):
+ Description of the commit that will be created.
+ Additionally, description of the PR if a PR is created (`create_pr` is True).
+ private (`bool`, *optional*):
+ Whether to make the repo private. If `None` (default), the repo will be public unless the
+ organization's default is private. This value is ignored if the repo already exists.
+ token (`str`, *optional*):
+ An optional authentication token for the Hugging Face Hub. If no token is passed, will default
+ to the token saved locally when logging in with `huggingface-cli login`. Will raise an error
+ if no token is passed and the user is not logged-in.
+ revision (`str`, *optional*):
+ Branch to push the uploaded files to. Defaults to the `"main"` branch.
+ create_pr (`bool`, *optional*, defaults to `False`):
+ Whether to create a PR with the uploaded files or directly commit.
+ max_shard_size (`int` or `str`, *optional*):
+ Optional maximum size of the dataset shards to be uploaded to the hub. If expressed as a string, needs to be digits followed
+ by a unit (like `"5MB"`). If not provided, shard count defaults to this dataset's `.num_shards`.
+ num_shards (`int`, *optional*):
+ Number of shards to write. If `max_shard_size` is provided and `num_shards` is not, then the number of shards is estimated
+ from `max_shard_size`.
+ embed_external_files (`bool`, defaults to `True`):
+ Whether to embed file bytes in the shards.
+ In particular, this will do the following before the push for the fields of type:
+
+ - [`Audio`] and [`Image`]: remove local path information and embed file content in the Parquet files.
+ num_proc (`int`, *optional*, defaults to `None`):
+ Number of processes when preparing and uploading the dataset.
+ This is helpful if the dataset is made of many samples and transformations.
+ I uses "spawn" context to work with hf_xet, the rust client for fast uploads to HF.
+ Multiprocessing is disabled by default.
+
+ Return:
+ huggingface_hub.CommitInfo
+
+ Example:
+
+ ```python
+ >>> dataset.push_to_hub("/")
+ >>> dataset_dict.push_to_hub("/", private=True)
+ >>> dataset.push_to_hub("/", max_shard_size="1GB")
+ >>> dataset.push_to_hub("/", num_shards=1024)
+ ```
+
+ If your dataset has multiple splits (e.g. train/validation/test):
+
+ ```python
+ >>> train_dataset.push_to_hub("/", split="train")
+ >>> val_dataset.push_to_hub("/", split="validation")
+ >>> # later
+ >>> dataset = load_dataset("/")
+ >>> train_dataset = dataset["train"]
+ >>> val_dataset = dataset["validation"]
+ ```
+
+ If you want to add a new configuration (or subset) to a dataset (e.g. if the dataset has multiple tasks/versions/languages):
+
+ ```python
+ >>> english_dataset.push_to_hub("/", "en")
+ >>> french_dataset.push_to_hub("/", "fr")
+ >>> # later
+ >>> english_dataset = load_dataset("/", "en")
+ >>> french_dataset = load_dataset("/", "fr")
+ ```
+ """
+ if num_proc is not None and num_proc > self.num_shards:
+ logger.warning(
+ f"Too many num_proc: {num_proc} (max is dataset.num_shards={self.num_shards}). "
+ f"Stopping {num_proc - self.num_shards} processes."
+ )
+ logger.info(
+ f"To parallelize data loading, we give each process some shards (or data sources) to process. "
+ f"Therefore it's unnecessary to have a number of processes greater than dataset.num_shards={self.num_shards}. "
+ f"To enable more parallelism, please split the dataset in more files than {self.num_shards}."
+ )
+ num_proc = self.num_shards
+
+ if config_name == "data":
+ raise ValueError("`config_name` cannot be 'data'. Please, choose another name for configuration.")
+
+ if max_shard_size is not None and num_shards is not None:
+ raise ValueError(
+ "Failed to push_to_hub: please specify either max_shard_size or num_shards, but not both."
+ )
+
+ if split is None:
+ split = str(self.split) if self.split is not None else "train"
+
+ if not re.match(_split_re, split):
+ raise ValueError(f"Split name should match '{_split_re}' but got '{split}'.")
+
+ if not data_dir:
+ data_dir = config_name if config_name != "default" else "data" # for backward compatibility
+
+ api = HfApi(endpoint=config.HF_ENDPOINT, token=token, library_name="datasets", library_version=__version__)
+ if repo_id.startswith("buckets/"):
+ if BucketNotFoundError is None:
+ raise ImportError("Pushing datasets to buckets requires huggingface_hub>=1.6.0")
+ _, _namespace, _bucket_name, *_path_segments = repo_id.split("/")
+ try:
+ bucket_id = api.bucket_info(_namespace + "/" + _bucket_name).id
+ except BucketNotFoundError:
+ bucket_url = api.create_bucket(_namespace + "/" + _bucket_name, private=private, exist_ok=True)
+ bucket_id = bucket_url.bucket_id
+ path = "/".join(s for s in _path_segments if s)
+ return _push_to_bucket(
+ self,
+ bucket_id=bucket_id,
+ path=path,
+ config_name=config_name,
+ set_default=set_default,
+ split=split,
+ data_dir=data_dir,
+ token=token,
+ max_shard_size=max_shard_size,
+ num_shards=num_shards,
+ embed_external_files=embed_external_files,
+ num_proc=num_proc,
+ )
+ else:
+ try:
+ repo_id = api.repo_info(repo_id, repo_type="dataset").id
+ except RepositoryNotFoundError:
+ repo_url = api.create_repo(
+ repo_id,
+ repo_type="dataset",
+ private=private,
+ exist_ok=True,
+ )
+ repo_id = repo_url.repo_id
+
+ if revision is not None and not revision.startswith("refs/pr/"):
+ # We do not call create_branch for a PR reference: 400 Bad Request
+ api.create_branch(repo_id, branch=revision, repo_type="dataset", exist_ok=True)
+ return _push_to_repo(
+ self,
+ repo_id=repo_id,
+ config_name=config_name,
+ set_default=set_default,
+ split=split,
+ data_dir=data_dir,
+ commit_message=commit_message,
+ commit_description=commit_description,
+ token=token,
+ revision=revision,
+ create_pr=create_pr,
+ max_shard_size=max_shard_size,
+ num_shards=num_shards,
+ embed_external_files=embed_external_files,
+ num_proc=num_proc,
+ )
+
+
+def _concatenate_iterable_datasets(
+ dsets: list[IterableDataset],
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ axis: int = 0,
+) -> IterableDataset:
+ """
+ Converts a list of `IterableDataset` with the same schema into a single `IterableDataset`.
+ Missing data are filled with None values.
+
+
+
+ Args:
+ dsets (`List[datasets.IterableDataset]`): List of Datasets to concatenate.
+ info (`DatasetInfo`, optional): Dataset information, like description, citation, etc.
+ split (`NamedSplit`, optional): Name of the dataset split.
+ axis (``{0, 1}``, default ``0``, meaning over rows):
+ Axis to concatenate over, where ``0`` means over rows (vertically) and ``1`` means over columns
+ (horizontally).
+
+ *New in version 1.6.0*
+
+ Example:
+
+ ```py
+ >>> ds3 = _concatenate_iterable_datasets([ds1, ds2])
+ ```
+ """
+ dsets = [d._resolve_features() for d in dsets]
+
+ # Perform checks (and a potentional cast if axis=0)
+ if axis == 0:
+ _check_if_features_can_be_aligned([dset.features for dset in dsets])
+ else:
+ _check_column_names([col_name for dset in dsets for col_name in dset.features])
+
+ # Check format is consistent; if so, will set format for concatenated dataset
+ if all(dset._formatting is None for dset in dsets):
+ formatting = None
+ elif any(dset._formatting is None for dset in dsets):
+ formatting = None
+ logger.info(
+ "Some of the datasets have disparate format or format not set. Resetting the format of the concatenated dataset."
+ )
+ else:
+ format_type_set = {dset._formatting.format_type for dset in dsets}
+ if len(format_type_set) == 1:
+ format_type = format_type_set.pop()
+ formatting = FormattingConfig(format_type=format_type)
+ else:
+ formatting = None
+ logger.info(
+ "Some of the datasets have disparate format or format not set. Resetting the format of the concatenated dataset."
+ )
+
+ # TODO: improve this to account for a mix of ClassLabel and Value for example
+ # right now it would keep the type of the first dataset in the list
+ features = Features(
+ {k: v for features in _align_features([dset.features for dset in dsets]) for k, v in features.items()}
+ )
+
+ ex_iterables = [deepcopy(d._ex_iterable) for d in dsets]
+ if axis == 0:
+ ex_iterable = VerticallyConcatenatedMultiSourcesExamplesIterable(ex_iterables)
+ else:
+ if all(ex_iterable.iter_arrow for ex_iterable in ex_iterables):
+ from .arrow_writer import get_arrow_writer_batch_size_from_features
+
+ batch_size = get_arrow_writer_batch_size_from_features(features) or config.DEFAULT_MAX_BATCH_SIZE
+ ex_iterables = [
+ RebatchedArrowExamplesIterable(ex_iterable, batch_size=batch_size) for ex_iterable in ex_iterables
+ ]
+ ex_iterable = HorizontallyConcatenatedMultiSourcesExamplesIterable(ex_iterables)
+ # Set new info - we update the features
+ # setting the features also ensures to fill missing columns with None
+ if info is None:
+ info = DatasetInfo.from_merge([d.info for d in dsets])
+ else:
+ info = info.copy()
+ info.features = features
+ # Get all the auth tokens per repository - in case the datasets come from different private repositories
+ token_per_repo_id = {repo_id: token for dataset in dsets for repo_id, token in dataset._token_per_repo_id.items()}
+ # Return new daset
+ return IterableDataset(
+ ex_iterable=ex_iterable,
+ info=info,
+ split=split,
+ token_per_repo_id=token_per_repo_id,
+ formatting=formatting,
+ )
+
+
+def _interleave_iterable_datasets(
+ datasets: list[IterableDataset],
+ probabilities: Optional[list[float]] = None,
+ seed: Optional[int] = None,
+ info: Optional[DatasetInfo] = None,
+ split: Optional[NamedSplit] = None,
+ stopping_strategy: Literal[
+ "first_exhausted", "all_exhausted", "all_exhausted_without_replacement"
+ ] = "first_exhausted",
+) -> IterableDataset:
+ """
+ Interleave several iterable datasets (sources) into a single iterable dataset.
+ The new iterable dataset alternates between the sources to yield examples.
+ If `probabilities = None` (default) the iterable dataset will cycles through the sources in order for each next example in the iteration.
+ If `probabilities` is not `None, the iterable dataset will sample a random source according to the provided probabilities for each next examples in the iteration.
+
+
+
+ Args:
+ datasets (`List[IterableDataset]`): list of datasets to interleave
+ probabilities (`List[float]`, optional, default None): If specified, the new iterable dataset samples
+ examples from one source at a time according to these probabilities.
+ seed (`int`, optional, default None): The random seed used to choose a source for each example.
+ stopping_strategy (`str`, defaults to `first_exhausted`):
+ Two strategies are proposed right now.
+ By default, `first_exhausted` is an undersampling strategy, i.e the dataset construction is stopped as soon as one dataset has ran out of samples.
+ If the strategy is `all_exhausted`, we use an oversampling strategy, i.e the dataset construction is stopped as soon as every samples of every dataset has been added at least once.
+ Note that if the strategy is `all_exhausted`, the interleaved dataset size can get enormous:
+ - with no probabilities, the resulting dataset will have max_length_datasets*nb_dataset samples.
+ - with given probabilities, the resulting dataset will have more samples if some datasets have really low probability of visiting.
+
+ Output:
+ `datasets.IterableDataset`
+ """
+ datasets = [d._resolve_features() for d in datasets]
+
+ # Perform checks
+ _check_if_features_can_be_aligned([dset.features for dset in datasets])
+ for i, dset in enumerate(datasets):
+ if datasets[0]._distributed != dset._distributed:
+ raise ValueError(
+ f"Datasets should be identically split_by_node before interleaving, but got {datasets[0]._distributed}!={dset._distributed} at index 0 and {i}"
+ )
+
+ # TODO: improve this to account for a mix of ClassLabel and Value for example
+ # right now it would keep the type of the first dataset in the list
+ features = Features(
+ {k: v for features in _align_features([dset.features for dset in datasets]) for k, v in features.items()}
+ )
+
+ ex_iterables = [deepcopy(d._ex_iterable) for d in datasets]
+ if all(ex_iterable.iter_arrow for ex_iterable in ex_iterables):
+ ex_iterables = [RebatchedArrowExamplesIterable(ex_iterable, batch_size=1) for ex_iterable in ex_iterables]
+ # Use cycling or random cycling of sources
+ if probabilities is None:
+ ex_iterable = CyclingMultiSourcesExamplesIterable(ex_iterables, stopping_strategy=stopping_strategy)
+ else:
+ generator = np.random.default_rng(seed)
+ ex_iterable = RandomlyCyclingMultiSourcesExamplesIterable(
+ ex_iterables,
+ generator=generator,
+ probabilities=probabilities,
+ stopping_strategy=stopping_strategy,
+ )
+ # Set new info - we update the features
+ # setting the features also ensures to fill missing columns with None
+ if info is None:
+ info = DatasetInfo.from_merge([d.info for d in datasets])
+ else:
+ info = info.copy()
+ info.features = features
+ # Get all the auth tokens per repository - in case the datasets come from different private repositories
+ token_per_repo_id = {
+ repo_id: token for dataset in datasets for repo_id, token in dataset._token_per_repo_id.items()
+ }
+ # Return new daset
+ return IterableDataset(
+ ex_iterable=ex_iterable,
+ info=info,
+ split=split,
+ token_per_repo_id=token_per_repo_id,
+ distributed=datasets[0]._distributed,
+ )
+
+
+def _split_by_node_iterable_dataset(dataset: IterableDataset, rank: int, world_size: int) -> IterableDataset:
+ """
+ Split an iterable dataset for the node at rank `rank` in a pool of nodes of size `world_size`.
+
+ If the dataset has a number of shards that is a factor of `world_size` (i.e. if `dataset.num_shards % world_size == 0`),
+ then the shards are evenly assigned across the nodes, which is the most optimized.
+ Otherwise, each node keeps 1 example out of `world_size`, skipping the other examples.
+
+ Args:
+ dataset ([`IterableDataset`]):
+ The iterable dataset to split by node.
+ rank (`int`):
+ Rank of the current node.
+ world_size (`int`):
+ Total number of nodes.
+
+ Returns:
+ [`IterableDataset`]: The iterable dataset to be used on the node at rank `rank`.
+ """
+ if dataset._distributed:
+ rank = world_size * dataset._distributed.rank + rank
+ world_size = world_size * dataset._distributed.world_size
+ distributed = DistributedConfig(rank=rank, world_size=world_size)
+ return IterableDataset(
+ ex_iterable=dataset._ex_iterable,
+ info=dataset._info.copy(),
+ split=dataset._split,
+ formatting=dataset._formatting,
+ distributed=distributed,
+ token_per_repo_id=dataset._token_per_repo_id,
+ )
+
+
+async def _apply_async(pool, func, x):
+ future = pool.apply_async(func, (x,))
+ while True:
+ if future.ready():
+ return future.get()
+ else:
+ await asyncio.sleep(0)
+
+
+def _batch_fn(unbatched):
+ return {k: [v] for k, v in unbatched.items()}
+
+
+def _generate_tables_from_polars(df: Union["pl.DataFrame", "pl.LazyFrame"]) -> Iterator[tuple["BuilderKey", pa.Table]]:
+ import polars as pl
+
+ from .builder import Key as BuilderKey
+
+ for slice_idx, df_slice in enumerate(df.collect_batches() if isinstance(df, pl.LazyFrame) else df.iter_slices()):
+ yield BuilderKey(0, slice_idx), df_slice.to_arrow()
diff --git a/venv/Lib/site-packages/datasets/load.py b/venv/Lib/site-packages/datasets/load.py
new file mode 100644
index 0000000000000000000000000000000000000000..0973093c5448a8cba2c47a790c78afc7b811a74f
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/load.py
@@ -0,0 +1,1782 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Access datasets."""
+
+import glob
+import importlib
+import inspect
+import json
+import os
+import posixpath
+from collections import Counter
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal, Optional, Union, overload
+
+import fsspec
+import httpx
+import requests
+import yaml
+from fsspec.core import url_to_fs
+from huggingface_hub import DatasetCard, DatasetCardData, HfApi, HfFileSystem
+from huggingface_hub.utils import (
+ EntryNotFoundError,
+ GatedRepoError,
+ LocalEntryNotFoundError,
+ OfflineModeIsEnabled,
+ RepositoryNotFoundError,
+ RevisionNotFoundError,
+ get_session,
+)
+from packaging import version
+
+from . import __version__, config
+from .arrow_dataset import Dataset
+from .builder import BuilderConfig, DatasetBuilder
+from .data_files import (
+ DataFilesDict,
+ DataFilesList,
+ DataFilesPatternsDict,
+ EmptyDatasetError,
+ get_data_patterns,
+ sanitize_patterns,
+)
+from .dataset_dict import DatasetDict, IterableDatasetDict
+from .download.download_config import DownloadConfig
+from .download.download_manager import DownloadMode
+from .download.streaming_download_manager import StreamingDownloadManager, xbasename, xglob, xjoin
+from .exceptions import DataFilesNotFoundError, DatasetNotFoundError
+from .features import Features
+from .features.features import _fix_for_backward_compatible_features
+from .fingerprint import Hasher
+from .info import DatasetInfo, DatasetInfosDict
+from .iterable_dataset import IterableDataset
+from .naming import camelcase_to_snakecase, snakecase_to_camelcase
+from .packaged_modules import (
+ _ALL_ALLOWED_EXTENSIONS,
+ _ALL_METADATA_FILENAMES,
+ _EXTENSION_TO_MODULE,
+ _MODULE_TO_EXTENSIONS,
+ _MODULE_TO_METADATA_EXTENSIONS,
+ _MODULE_TO_METADATA_FILE_NAMES,
+ _PACKAGED_DATASETS_MODULES,
+)
+from .splits import Split
+from .utils import _dataset_viewer
+from .utils.file_utils import (
+ _raise_if_offline_mode_is_enabled,
+ cached_path,
+ get_datasets_user_agent,
+ is_relative_path,
+ relative_to_absolute_path,
+)
+from .utils.hub import hf_dataset_url
+from .utils.info_utils import VerificationMode, is_small_dataset
+from .utils.logging import get_logger
+from .utils.metadata import MetadataConfigs
+from .utils.typing import PathLike
+from .utils.version import Version
+
+
+if config.HF_HUB_VERSION >= version.parse("1.6.0"):
+ from huggingface_hub.errors import BucketNotFoundError
+
+else:
+ BucketNotFoundError = None
+
+
+logger = get_logger(__name__)
+
+
+class _InitializeConfiguredDatasetBuilder:
+ """
+ From https://stackoverflow.com/questions/4647566/pickle-a-dynamically-parameterized-sub-class
+ See also ConfiguredDatasetBuilder.__reduce__
+ When called with the param value as the only argument, returns an
+ un-initialized instance of the parameterized class. Subsequent __setstate__
+ will be called by pickle.
+ """
+
+ def __call__(self, builder_cls, metadata_configs, default_config_name, name):
+ # make a simple object which has no complex __init__ (this one will do)
+ obj = _InitializeConfiguredDatasetBuilder()
+ obj.__class__ = configure_builder_class(
+ builder_cls, metadata_configs, default_config_name=default_config_name, dataset_name=name
+ )
+ return obj
+
+
+def configure_builder_class(
+ builder_cls: type[DatasetBuilder],
+ builder_configs: list[BuilderConfig],
+ default_config_name: Optional[str],
+ dataset_name: str,
+) -> type[DatasetBuilder]:
+ """
+ Dynamically create a builder class with custom builder configs parsed from README.md file,
+ i.e. set BUILDER_CONFIGS class variable of a builder class to custom configs list.
+ """
+
+ class ConfiguredDatasetBuilder(builder_cls):
+ BUILDER_CONFIGS = builder_configs
+ DEFAULT_CONFIG_NAME = default_config_name
+
+ __module__ = builder_cls.__module__ # so that the actual packaged builder can be imported
+
+ def __reduce__(self): # to make dynamically created class pickable, see _InitializeParameterizedDatasetBuilder
+ parent_builder_cls = self.__class__.__mro__[1]
+ return (
+ _InitializeConfiguredDatasetBuilder(),
+ (
+ parent_builder_cls,
+ self.BUILDER_CONFIGS,
+ self.DEFAULT_CONFIG_NAME,
+ self.dataset_name,
+ ),
+ self.__dict__.copy(),
+ )
+
+ ConfiguredDatasetBuilder.__name__ = (
+ f"{builder_cls.__name__.lower().capitalize()}{snakecase_to_camelcase(dataset_name)}"
+ )
+ ConfiguredDatasetBuilder.__qualname__ = (
+ f"{builder_cls.__name__.lower().capitalize()}{snakecase_to_camelcase(dataset_name)}"
+ )
+
+ return ConfiguredDatasetBuilder
+
+
+def import_main_class(module_path) -> Optional[type[DatasetBuilder]]:
+ """Import a module at module_path and return its main class: a DatasetBuilder"""
+ module = importlib.import_module(module_path)
+ # Find the main class in our imported module
+ module_main_cls = None
+ for name, obj in module.__dict__.items():
+ if inspect.isclass(obj) and issubclass(obj, DatasetBuilder):
+ if inspect.isabstract(obj):
+ continue
+ module_main_cls = obj
+ obj_module = inspect.getmodule(obj)
+ if obj_module is not None and module == obj_module:
+ break
+
+ return module_main_cls
+
+
+def get_dataset_builder_class(
+ dataset_module: "DatasetModule", dataset_name: Optional[str] = None
+) -> type[DatasetBuilder]:
+ builder_cls = import_main_class(dataset_module.module_path)
+ if dataset_module.builder_configs_parameters.builder_configs:
+ dataset_name = dataset_name or dataset_module.builder_kwargs.get("dataset_name")
+ if dataset_name is None:
+ raise ValueError("dataset_name should be specified but got None")
+ builder_cls = configure_builder_class(
+ builder_cls,
+ builder_configs=dataset_module.builder_configs_parameters.builder_configs,
+ default_config_name=dataset_module.builder_configs_parameters.default_config_name,
+ dataset_name=dataset_name,
+ )
+ return builder_cls
+
+
+def increase_load_count(name: str):
+ """Update the download count of a dataset."""
+ if not config.HF_HUB_OFFLINE and config.HF_UPDATE_DOWNLOAD_COUNTS:
+ try:
+ get_session().head(
+ "/".join((config.S3_DATASETS_BUCKET_PREFIX, name, name + ".py")),
+ headers={"User-Agent": get_datasets_user_agent()},
+ timeout=3,
+ )
+ except Exception:
+ pass
+
+
+def infer_module_for_data_files_list(
+ data_files_list: DataFilesList, download_config: Optional[DownloadConfig] = None
+) -> tuple[Optional[str], dict]:
+ """Infer module (and builder kwargs) from list of data files.
+
+ It picks the module based on the most common file extension.
+ In case of a draw ".parquet" is the favorite, and then alphabetical order.
+
+ Args:
+ data_files_list (DataFilesList): List of data files.
+ download_config (bool or str, optional): Mainly use `token` or `storage_options` to support different platforms and auth types.
+
+ Returns:
+ tuple[str, dict[str, Any]]: Tuple with
+ - inferred module name
+ - dict of builder kwargs
+ """
+ extensions_counter = Counter(
+ ("." + suffix.lower(), xbasename(filepath) in _ALL_METADATA_FILENAMES)
+ for filepath in data_files_list
+ for suffix in xbasename(filepath).split(".")[1:]
+ )
+ if extensions_counter:
+
+ def sort_key(ext_count: tuple[tuple[str, bool], int]) -> tuple[int, bool]:
+ """Sort by count and set ".parquet" as the favorite in case of a draw, and ignore metadata files"""
+ (ext, is_metadata), count = ext_count
+ return (
+ not is_metadata,
+ count,
+ ext == ".parquet",
+ ext == ".lance",
+ ext == ".arrow",
+ ext == ".jsonl",
+ ext == ".json",
+ ext == ".csv",
+ ext,
+ )
+
+ for (ext, _), _ in sorted(extensions_counter.items(), key=sort_key, reverse=True):
+ if ext in _EXTENSION_TO_MODULE:
+ return _EXTENSION_TO_MODULE[ext]
+ elif ext == ".zip":
+ return infer_module_for_data_files_list_in_archives(data_files_list, download_config=download_config)
+ return None, {}
+
+
+def infer_module_for_data_files_list_in_archives(
+ data_files_list: DataFilesList, download_config: Optional[DownloadConfig] = None
+) -> tuple[Optional[str], dict]:
+ """Infer module (and builder kwargs) from list of archive data files.
+
+ Args:
+ data_files_list (DataFilesList): List of data files.
+ download_config (bool or str, optional): Mainly use `token` or `storage_options` to support different platforms and auth types.
+
+ Returns:
+ tuple[str, dict[str, Any]]: Tuple with
+ - inferred module name
+ - dict of builder kwargs
+ """
+ archived_files = []
+ archive_files_counter = 0
+ for filepath in data_files_list:
+ if str(filepath).endswith(".zip"):
+ archive_files_counter += 1
+ if archive_files_counter > config.ARCHIVES_MAX_NUMBER_FOR_MODULE_INFERENCE:
+ break
+ extracted = xjoin(StreamingDownloadManager().extract(filepath), "**")
+ archived_files += [
+ f.split("::")[0] for f in xglob(extracted, recursive=True, download_config=download_config)
+ ]
+ extensions_counter = Counter(
+ "." + suffix.lower() for filepath in archived_files for suffix in xbasename(filepath).split(".")[1:]
+ )
+ if extensions_counter:
+ most_common = extensions_counter.most_common(1)[0][0]
+ if most_common in _EXTENSION_TO_MODULE:
+ return _EXTENSION_TO_MODULE[most_common]
+ return None, {}
+
+
+def infer_module_for_data_files(
+ data_files: DataFilesDict, path: Optional[str] = None, download_config: Optional[DownloadConfig] = None
+) -> tuple[Optional[str], dict[str, Any]]:
+ """Infer module (and builder kwargs) from data files. Raise if module names for different splits don't match.
+
+ Args:
+ data_files ([`DataFilesDict`]): Dict of list of data files.
+ path (str, *optional*): Dataset name or path.
+ download_config ([`DownloadConfig`], *optional*):
+ Specific download configuration parameters to authenticate on the Hugging Face Hub for private remote files.
+
+ Returns:
+ tuple[str, dict[str, Any]]: Tuple with
+ - inferred module name
+ - builder kwargs
+ """
+ split_modules = {
+ split: infer_module_for_data_files_list(data_files_list, download_config=download_config)
+ for split, data_files_list in data_files.items()
+ }
+ module_name, default_builder_kwargs = next(iter(split_modules.values()))
+ if any((module_name, default_builder_kwargs) != split_module for split_module in split_modules.values()):
+ raise ValueError(f"Couldn't infer the same data file format for all splits. Got {split_modules}")
+ if not module_name:
+ raise DataFilesNotFoundError("No (supported) data files found" + (f" in {path}" if path else ""))
+ return module_name, default_builder_kwargs
+
+
+def create_builder_configs_from_metadata_configs(
+ module_path: str,
+ metadata_configs: MetadataConfigs,
+ base_path: Optional[str] = None,
+ default_builder_kwargs: dict[str, Any] = None,
+ download_config: Optional[DownloadConfig] = None,
+) -> tuple[list[BuilderConfig], str]:
+ builder_cls = import_main_class(module_path)
+ builder_config_cls = builder_cls.BUILDER_CONFIG_CLASS
+ default_config_name = metadata_configs.get_default_config_name()
+ builder_configs = []
+ default_builder_kwargs = {} if default_builder_kwargs is None else default_builder_kwargs
+
+ base_path = base_path if base_path is not None else ""
+ for config_name, config_params in metadata_configs.items():
+ config_data_files = config_params.get("data_files")
+ config_data_dir = config_params.get("data_dir")
+ config_base_path = xjoin(base_path, config_data_dir) if config_data_dir else base_path
+ try:
+ config_patterns = (
+ sanitize_patterns(config_data_files)
+ if config_data_files is not None
+ else get_data_patterns(config_base_path, download_config=download_config)
+ )
+ config_data_files_dict = DataFilesPatternsDict.from_patterns(
+ config_patterns,
+ allowed_extensions=_ALL_ALLOWED_EXTENSIONS,
+ )
+ except EmptyDatasetError as e:
+ raise EmptyDatasetError(
+ f"Dataset at '{base_path}' doesn't contain data files matching the patterns for config '{config_name}',"
+ f" check `data_files` and `data_fir` parameters in the `configs` YAML field in README.md. "
+ ) from e
+ ignored_params = [
+ param for param in config_params if not hasattr(builder_config_cls, param) and param != "default"
+ ]
+ if ignored_params:
+ logger.warning(
+ f"Some datasets params were ignored: {ignored_params}. "
+ "Make sure to use only valid params for the dataset builder and to have "
+ "a up-to-date version of the `datasets` library."
+ )
+ builder_configs.append(
+ builder_config_cls(
+ name=config_name,
+ data_files=config_data_files_dict,
+ data_dir=config_data_dir,
+ **{
+ param: value
+ for param, value in {**default_builder_kwargs, **config_params}.items()
+ if hasattr(builder_config_cls, param) and param not in ("default", "data_files", "data_dir")
+ },
+ )
+ )
+ return builder_configs, default_config_name
+
+
+@dataclass
+class BuilderConfigsParameters:
+ """Dataclass containing objects related to creation of builder configurations from yaml's metadata content.
+
+ Attributes:
+ metadata_configs (`MetadataConfigs`, *optional*):
+ Configs parsed from yaml's metadata.
+ builder_configs (`list[BuilderConfig]`, *optional*):
+ List of BuilderConfig objects created from metadata_configs above.
+ default_config_name (`str`):
+ Name of default config taken from yaml's metadata.
+ """
+
+ metadata_configs: Optional[MetadataConfigs] = None
+ builder_configs: Optional[list[BuilderConfig]] = None
+ default_config_name: Optional[str] = None
+
+
+@dataclass
+class DatasetModule:
+ module_path: str
+ hash: str
+ builder_kwargs: dict
+ builder_configs_parameters: BuilderConfigsParameters = field(default_factory=BuilderConfigsParameters)
+ dataset_infos: Optional[DatasetInfosDict] = None
+
+
+class _DatasetModuleFactory:
+ def get_module(self) -> DatasetModule:
+ raise NotImplementedError
+
+
+class LocalDatasetModuleFactory(_DatasetModuleFactory):
+ """Get the module of a dataset loaded from the user's data files. The dataset builder module to use is inferred
+ from the data files extensions."""
+
+ def __init__(
+ self,
+ path: str,
+ data_dir: Optional[str] = None,
+ data_files: Optional[Union[str, list, dict]] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ ):
+ if data_dir and os.path.isabs(data_dir):
+ raise ValueError(f"`data_dir` must be relative to a dataset directory's root: {path}")
+
+ self.path = Path(path).as_posix()
+ self.name = Path(path).stem
+ self.data_files = data_files
+ self.data_dir = data_dir
+ self.download_mode = download_mode
+
+ def get_module(self) -> DatasetModule:
+ readme_path = os.path.join(self.path, config.REPOCARD_FILENAME)
+ standalone_yaml_path = os.path.join(self.path, config.REPOYAML_FILENAME)
+ dataset_card_data = DatasetCard.load(readme_path).data if os.path.isfile(readme_path) else DatasetCardData()
+ if os.path.exists(standalone_yaml_path):
+ with open(standalone_yaml_path, encoding="utf-8") as f:
+ standalone_yaml_data = yaml.safe_load(f.read())
+ if standalone_yaml_data:
+ _dataset_card_data_dict = dataset_card_data.to_dict()
+ _dataset_card_data_dict.update(standalone_yaml_data)
+ dataset_card_data = DatasetCardData(**_dataset_card_data_dict)
+ metadata_configs = MetadataConfigs.from_dataset_card_data(dataset_card_data)
+ dataset_infos = DatasetInfosDict.from_dataset_card_data(dataset_card_data)
+ # we need a set of data files to find which dataset builder to use
+ # because we need to infer module name by files extensions
+ base_path = Path(self.path, self.data_dir or "").expanduser().resolve().as_posix()
+ if self.data_files is not None:
+ patterns = sanitize_patterns(self.data_files)
+ elif metadata_configs and not self.data_dir and "data_files" in next(iter(metadata_configs.values())):
+ patterns = sanitize_patterns(next(iter(metadata_configs.values()))["data_files"])
+ else:
+ patterns = get_data_patterns(base_path)
+ data_files = DataFilesDict.from_patterns(
+ patterns,
+ base_path=base_path,
+ allowed_extensions=_ALL_ALLOWED_EXTENSIONS,
+ )
+ module_name, default_builder_kwargs = infer_module_for_data_files(
+ data_files=data_files,
+ path=self.path,
+ )
+ data_files = data_files.filter(
+ extensions=_MODULE_TO_EXTENSIONS[module_name] + _MODULE_TO_METADATA_EXTENSIONS[module_name],
+ file_names=_MODULE_TO_METADATA_FILE_NAMES[module_name],
+ )
+ module_path, _ = _PACKAGED_DATASETS_MODULES[module_name]
+ if metadata_configs:
+ builder_configs, default_config_name = create_builder_configs_from_metadata_configs(
+ module_path,
+ metadata_configs,
+ base_path=base_path,
+ default_builder_kwargs=default_builder_kwargs,
+ )
+ else:
+ builder_configs: list[BuilderConfig] = [
+ import_main_class(module_path).BUILDER_CONFIG_CLASS(
+ data_files=data_files,
+ **default_builder_kwargs,
+ )
+ ]
+ default_config_name = None
+ builder_kwargs = {
+ "base_path": self.path,
+ "dataset_name": camelcase_to_snakecase(Path(self.path).name),
+ }
+ if self.data_dir:
+ builder_kwargs["data_files"] = data_files
+ # this file is deprecated and was created automatically in old versions of push_to_hub
+ if os.path.isfile(os.path.join(self.path, config.DATASETDICT_INFOS_FILENAME)):
+ with open(os.path.join(self.path, config.DATASETDICT_INFOS_FILENAME), encoding="utf-8") as f:
+ legacy_dataset_infos = DatasetInfosDict(
+ {
+ config_name: DatasetInfo.from_dict(dataset_info_dict)
+ for config_name, dataset_info_dict in json.load(f).items()
+ }
+ )
+ if len(legacy_dataset_infos) == 1:
+ # old config e.g. named "username--dataset_name"
+ legacy_config_name = next(iter(legacy_dataset_infos))
+ legacy_dataset_infos["default"] = legacy_dataset_infos.pop(legacy_config_name)
+ legacy_dataset_infos.update(dataset_infos)
+ dataset_infos = legacy_dataset_infos
+ if default_config_name is None and len(dataset_infos) == 1:
+ default_config_name = next(iter(dataset_infos))
+
+ hash = Hasher.hash({"dataset_infos": dataset_infos, "builder_configs": builder_configs})
+ return DatasetModule(
+ module_path,
+ hash,
+ builder_kwargs,
+ dataset_infos=dataset_infos,
+ builder_configs_parameters=BuilderConfigsParameters(
+ metadata_configs=metadata_configs,
+ builder_configs=builder_configs,
+ default_config_name=default_config_name,
+ ),
+ )
+
+
+class PackagedDatasetModuleFactory(_DatasetModuleFactory):
+ """Get the dataset builder module from the ones that are packaged with the library: csv, json, etc."""
+
+ def __init__(
+ self,
+ name: str,
+ data_dir: Optional[str] = None,
+ data_files: Optional[Union[str, list, dict]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ ):
+ self.name = name
+ self.data_files = data_files
+ self.data_dir = data_dir
+ self.download_config = download_config
+ self.download_mode = download_mode
+ increase_load_count(name)
+
+ def get_module(self) -> DatasetModule:
+ base_path = Path(self.data_dir or "").expanduser().resolve().as_posix()
+ patterns = (
+ sanitize_patterns(self.data_files)
+ if self.data_files is not None
+ else get_data_patterns(base_path, download_config=self.download_config)
+ )
+ data_files = DataFilesDict.from_patterns(
+ patterns,
+ download_config=self.download_config,
+ base_path=base_path,
+ )
+
+ module_path, hash = _PACKAGED_DATASETS_MODULES[self.name]
+
+ builder_kwargs = {
+ "data_files": data_files,
+ "dataset_name": self.name,
+ }
+
+ return DatasetModule(module_path, hash, builder_kwargs)
+
+
+class HubDatasetModuleFactory(_DatasetModuleFactory):
+ """
+ Get the module of a dataset loaded from data files of a dataset repository.
+ The dataset builder module to use is inferred from the data files extensions.
+ """
+
+ def __init__(
+ self,
+ name: str,
+ commit_hash: str,
+ data_dir: Optional[str] = None,
+ data_files: Optional[Union[str, list, dict]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ use_exported_dataset_infos: bool = False,
+ ):
+ self.name = name
+ self.commit_hash = commit_hash
+ self.data_files = data_files
+ self.data_dir = data_dir
+ self.download_config = download_config or DownloadConfig()
+ self.download_mode = download_mode
+ self.use_exported_dataset_infos = use_exported_dataset_infos
+ increase_load_count(name)
+
+ def get_module(self) -> DatasetModule:
+ # Get the Dataset Card and fix the revision in case there are new commits in the meantime
+ api = HfApi(
+ endpoint=config.HF_ENDPOINT,
+ token=self.download_config.token,
+ library_name="datasets",
+ library_version=__version__,
+ user_agent=get_datasets_user_agent(self.download_config.user_agent),
+ )
+ try:
+ dataset_readme_path = api.hf_hub_download(
+ repo_id=self.name,
+ filename=config.REPOCARD_FILENAME,
+ repo_type="dataset",
+ revision=self.commit_hash,
+ proxies=self.download_config.proxies,
+ )
+ dataset_card_data = DatasetCard.load(dataset_readme_path).data
+ except EntryNotFoundError:
+ dataset_card_data = DatasetCardData()
+ download_config = self.download_config.copy()
+ if download_config.download_desc is None:
+ download_config.download_desc = "Downloading standalone yaml"
+ try:
+ standalone_yaml_path = cached_path(
+ hf_dataset_url(self.name, config.REPOYAML_FILENAME, revision=self.commit_hash),
+ download_config=download_config,
+ )
+ with open(standalone_yaml_path, encoding="utf-8") as f:
+ standalone_yaml_data = yaml.safe_load(f.read())
+ if standalone_yaml_data:
+ _dataset_card_data_dict = dataset_card_data.to_dict()
+ _dataset_card_data_dict.update(standalone_yaml_data)
+ dataset_card_data = DatasetCardData(**_dataset_card_data_dict)
+ except FileNotFoundError:
+ pass
+ base_path = f"hf://datasets/{self.name}@{self.commit_hash}/{self.data_dir or ''}".rstrip("/")
+ metadata_configs = MetadataConfigs.from_dataset_card_data(dataset_card_data)
+ dataset_infos = DatasetInfosDict.from_dataset_card_data(dataset_card_data)
+ if config.USE_PARQUET_EXPORT and self.use_exported_dataset_infos:
+ try:
+ exported_dataset_infos = _dataset_viewer.get_exported_dataset_infos(
+ dataset=self.name, commit_hash=self.commit_hash, token=self.download_config.token
+ )
+ exported_dataset_infos = DatasetInfosDict(
+ {
+ config_name: DatasetInfo.from_dict(exported_dataset_infos[config_name])
+ for config_name in exported_dataset_infos
+ }
+ )
+ except _dataset_viewer.DatasetViewerError:
+ exported_dataset_infos = None
+ else:
+ exported_dataset_infos = None
+ if exported_dataset_infos:
+ exported_dataset_infos.update(dataset_infos)
+ dataset_infos = exported_dataset_infos
+ # we need a set of data files to find which dataset builder to use
+ # because we need to infer module name by files extensions
+ if self.data_files is not None:
+ patterns = sanitize_patterns(self.data_files)
+ elif metadata_configs and not self.data_dir and "data_files" in next(iter(metadata_configs.values())):
+ patterns = sanitize_patterns(next(iter(metadata_configs.values()))["data_files"])
+ else:
+ patterns = get_data_patterns(base_path, download_config=self.download_config)
+ data_files = DataFilesDict.from_patterns(
+ patterns,
+ base_path=base_path,
+ allowed_extensions=_ALL_ALLOWED_EXTENSIONS,
+ download_config=self.download_config,
+ )
+ module_name, default_builder_kwargs = infer_module_for_data_files(
+ data_files=data_files,
+ path=self.name,
+ download_config=self.download_config,
+ )
+ data_files = data_files.filter(
+ extensions=_MODULE_TO_EXTENSIONS[module_name] + _MODULE_TO_METADATA_EXTENSIONS[module_name],
+ file_names=_MODULE_TO_METADATA_FILE_NAMES[module_name],
+ )
+ module_path, _ = _PACKAGED_DATASETS_MODULES[module_name]
+ if metadata_configs:
+ builder_configs, default_config_name = create_builder_configs_from_metadata_configs(
+ module_path,
+ metadata_configs,
+ base_path=base_path,
+ default_builder_kwargs=default_builder_kwargs,
+ download_config=self.download_config,
+ )
+ else:
+ builder_configs: list[BuilderConfig] = [
+ import_main_class(module_path).BUILDER_CONFIG_CLASS(
+ data_files=data_files,
+ **default_builder_kwargs,
+ )
+ ]
+ default_config_name = None
+ builder_kwargs = {
+ "base_path": base_path,
+ "repo_id": self.name,
+ "dataset_name": camelcase_to_snakecase(Path(self.name).name),
+ }
+ if self.data_dir:
+ builder_kwargs["data_files"] = data_files
+ download_config = self.download_config.copy()
+ if download_config.download_desc is None:
+ download_config.download_desc = "Downloading metadata"
+ try:
+ # this file is deprecated and was created automatically in old versions of push_to_hub
+ dataset_infos_path = cached_path(
+ hf_dataset_url(self.name, config.DATASETDICT_INFOS_FILENAME, revision=self.commit_hash),
+ download_config=download_config,
+ )
+ with open(dataset_infos_path, encoding="utf-8") as f:
+ legacy_dataset_infos = DatasetInfosDict(
+ {
+ config_name: DatasetInfo.from_dict(dataset_info_dict)
+ for config_name, dataset_info_dict in json.load(f).items()
+ }
+ )
+ if len(legacy_dataset_infos) == 1:
+ # old config e.g. named "username--dataset_name"
+ legacy_config_name = next(iter(legacy_dataset_infos))
+ legacy_dataset_infos["default"] = legacy_dataset_infos.pop(legacy_config_name)
+ legacy_dataset_infos.update(dataset_infos)
+ dataset_infos = legacy_dataset_infos
+ except FileNotFoundError:
+ pass
+ if default_config_name is None and len(dataset_infos) == 1:
+ default_config_name = next(iter(dataset_infos))
+
+ return DatasetModule(
+ module_path,
+ self.commit_hash,
+ builder_kwargs,
+ dataset_infos=dataset_infos,
+ builder_configs_parameters=BuilderConfigsParameters(
+ metadata_configs=metadata_configs,
+ builder_configs=builder_configs,
+ default_config_name=default_config_name,
+ ),
+ )
+
+
+class HubDatasetModuleFactoryWithParquetExport(_DatasetModuleFactory):
+ """
+ Get the module of a dataset loaded from parquet files of a dataset repository parquet export.
+ """
+
+ def __init__(
+ self,
+ name: str,
+ commit_hash: str,
+ download_config: Optional[DownloadConfig] = None,
+ ):
+ self.name = name
+ self.commit_hash = commit_hash
+ self.download_config = download_config or DownloadConfig()
+ increase_load_count(name)
+
+ def get_module(self) -> DatasetModule:
+ exported_parquet_files = _dataset_viewer.get_exported_parquet_files(
+ dataset=self.name, commit_hash=self.commit_hash, token=self.download_config.token
+ )
+ exported_dataset_infos = _dataset_viewer.get_exported_dataset_infos(
+ dataset=self.name, commit_hash=self.commit_hash, token=self.download_config.token
+ )
+ dataset_infos = DatasetInfosDict(
+ {
+ config_name: DatasetInfo.from_dict(exported_dataset_infos[config_name])
+ for config_name in exported_dataset_infos
+ }
+ )
+ parquet_commit_hash = (
+ HfApi(
+ endpoint=config.HF_ENDPOINT,
+ token=self.download_config.token,
+ library_name="datasets",
+ library_version=__version__,
+ user_agent=get_datasets_user_agent(self.download_config.user_agent),
+ )
+ .dataset_info(
+ self.name,
+ revision="refs/convert/parquet",
+ token=self.download_config.token,
+ timeout=100.0,
+ )
+ .sha
+ ) # fix the revision in case there are new commits in the meantime
+ metadata_configs = MetadataConfigs._from_exported_parquet_files_and_dataset_infos(
+ parquet_commit_hash=parquet_commit_hash,
+ exported_parquet_files=exported_parquet_files,
+ dataset_infos=dataset_infos,
+ )
+ module_path, _ = _PACKAGED_DATASETS_MODULES["parquet"]
+ builder_configs, default_config_name = create_builder_configs_from_metadata_configs(
+ module_path,
+ metadata_configs,
+ download_config=self.download_config,
+ )
+ builder_kwargs = {
+ "repo_id": self.name,
+ "dataset_name": camelcase_to_snakecase(Path(self.name).name),
+ }
+
+ return DatasetModule(
+ module_path,
+ self.commit_hash,
+ builder_kwargs,
+ dataset_infos=dataset_infos,
+ builder_configs_parameters=BuilderConfigsParameters(
+ metadata_configs=metadata_configs,
+ builder_configs=builder_configs,
+ default_config_name=default_config_name,
+ ),
+ )
+
+
+class CachedDatasetModuleFactory(_DatasetModuleFactory):
+ """
+ Get the module of a dataset that has been loaded once already and cached.
+ """
+
+ def __init__(
+ self,
+ name: str,
+ cache_dir: Optional[str] = None,
+ ):
+ self.name = name
+ self.cache_dir = cache_dir
+ assert self.name.count("/") <= 1
+
+ def get_module(self) -> DatasetModule:
+ cache_dir = os.path.expanduser(str(self.cache_dir or config.HF_DATASETS_CACHE))
+ namespace_and_dataset_name = self.name.split("/")
+ namespace_and_dataset_name[-1] = camelcase_to_snakecase(namespace_and_dataset_name[-1])
+ cached_relative_path = "___".join(namespace_and_dataset_name)
+ cached_datasets_directory_path_root = os.path.join(cache_dir, cached_relative_path)
+ cached_directory_paths = [
+ cached_directory_path
+ for cached_directory_path in glob.glob(os.path.join(cached_datasets_directory_path_root, "*", "*", "*"))
+ if os.path.isdir(cached_directory_path)
+ ]
+ if cached_directory_paths:
+ builder_kwargs = {
+ "repo_id": self.name,
+ "dataset_name": self.name.split("/")[-1],
+ }
+ warning_msg = f"Using the latest cached version of the dataset since {self.name} couldn't be found on the Hugging Face Hub"
+ if config.HF_HUB_OFFLINE:
+ warning_msg += " (offline mode is enabled)."
+ logger.warning(warning_msg)
+ return DatasetModule(
+ "datasets.packaged_modules.cache.cache",
+ "auto",
+ {**builder_kwargs, "version": "auto"},
+ )
+ raise FileNotFoundError(f"Dataset {self.name} is not cached in {self.cache_dir}")
+
+
+class HubBucketDatasetModuleFactory(_DatasetModuleFactory):
+ """
+ Get the module of a dataset loaded from data files of a a Storage Bucket.
+ The dataset builder module to use is inferred from the data files extensions.
+ """
+
+ def __init__(
+ self,
+ path: str,
+ data_dir: Optional[str] = None,
+ data_files: Optional[Union[str, list, dict]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ ):
+ self.path = Path(path).as_posix()
+ self.name = Path(path).stem
+ self.data_files = data_files
+ self.data_dir = data_dir
+ self.download_config = download_config
+ self.download_mode = download_mode
+
+ def get_module(self) -> DatasetModule:
+ hffs = HfFileSystem(
+ endpoint=config.HF_ENDPOINT,
+ token=self.download_config.token,
+ )
+ readme_path = xjoin(self.path, config.REPOCARD_FILENAME)
+ standalone_yaml_path = xjoin(self.path, config.REPOYAML_FILENAME)
+ try:
+ dataset_card_data = DatasetCard(hffs.read_text(readme_path, newline="", encoding="utf-8"))
+ except FileNotFoundError:
+ dataset_card_data = DatasetCardData()
+ try:
+ standalone_yaml_data = yaml.safe_load(hffs.read_text(standalone_yaml_path, newline="", encoding="utf-8"))
+ except FileNotFoundError:
+ dataset_card_data = DatasetCardData()
+ if hffs.exists(standalone_yaml_path):
+ with hffs.open(standalone_yaml_path, "r", encoding="utf-8") as f:
+ standalone_yaml_data = yaml.safe_load(f.read())
+ if standalone_yaml_data:
+ _dataset_card_data_dict = dataset_card_data.to_dict()
+ _dataset_card_data_dict.update(standalone_yaml_data)
+ dataset_card_data = DatasetCardData(**_dataset_card_data_dict)
+ metadata_configs = MetadataConfigs.from_dataset_card_data(dataset_card_data)
+ dataset_infos = DatasetInfosDict.from_dataset_card_data(dataset_card_data)
+ # we need a set of data files to find which dataset builder to use
+ # because we need to infer module name by files extensions
+ base_path = "hf://" + Path(self.path, self.data_dir or "").as_posix()
+ if self.data_files is not None:
+ patterns = sanitize_patterns(self.data_files)
+ elif metadata_configs and not self.data_dir and "data_files" in next(iter(metadata_configs.values())):
+ patterns = sanitize_patterns(next(iter(metadata_configs.values()))["data_files"])
+ else:
+ patterns = get_data_patterns(base_path, download_config=self.download_config)
+ data_files = DataFilesDict.from_patterns(
+ patterns,
+ base_path=base_path,
+ allowed_extensions=_ALL_ALLOWED_EXTENSIONS,
+ )
+ module_name, default_builder_kwargs = infer_module_for_data_files(
+ data_files=data_files,
+ path=self.path,
+ )
+ data_files = data_files.filter(
+ extensions=_MODULE_TO_EXTENSIONS[module_name] + _MODULE_TO_METADATA_EXTENSIONS[module_name],
+ file_names=_MODULE_TO_METADATA_FILE_NAMES[module_name],
+ )
+ module_path, _ = _PACKAGED_DATASETS_MODULES[module_name]
+ if metadata_configs:
+ builder_configs, default_config_name = create_builder_configs_from_metadata_configs(
+ module_path,
+ metadata_configs,
+ base_path=base_path,
+ default_builder_kwargs=default_builder_kwargs,
+ )
+ else:
+ builder_configs: list[BuilderConfig] = [
+ import_main_class(module_path).BUILDER_CONFIG_CLASS(
+ data_files=data_files,
+ **default_builder_kwargs,
+ )
+ ]
+ default_config_name = None
+ builder_kwargs = {
+ "base_path": base_path,
+ "dataset_name": camelcase_to_snakecase(Path(self.path).name),
+ }
+ if self.data_dir:
+ builder_kwargs["data_files"] = data_files
+ # this file is deprecated and was created automatically in old versions of push_to_hub
+ if hffs.isfile(xjoin(self.path, config.DATASETDICT_INFOS_FILENAME)):
+ with hffs.open(xjoin(self.path, config.DATASETDICT_INFOS_FILENAME), "r", encoding="utf-8") as f:
+ legacy_dataset_infos = DatasetInfosDict(
+ {
+ config_name: DatasetInfo.from_dict(dataset_info_dict)
+ for config_name, dataset_info_dict in json.load(f).items()
+ }
+ )
+ if len(legacy_dataset_infos) == 1:
+ # old config e.g. named "username--dataset_name"
+ legacy_config_name = next(iter(legacy_dataset_infos))
+ legacy_dataset_infos["default"] = legacy_dataset_infos.pop(legacy_config_name)
+ legacy_dataset_infos.update(dataset_infos)
+ dataset_infos = legacy_dataset_infos
+ if default_config_name is None and len(dataset_infos) == 1:
+ default_config_name = next(iter(dataset_infos))
+
+ hash = Hasher.hash({"dataset_infos": dataset_infos, "builder_configs": builder_configs})
+ return DatasetModule(
+ module_path,
+ hash,
+ builder_kwargs,
+ dataset_infos=dataset_infos,
+ builder_configs_parameters=BuilderConfigsParameters(
+ metadata_configs=metadata_configs,
+ builder_configs=builder_configs,
+ default_config_name=default_config_name,
+ ),
+ )
+
+
+def dataset_module_factory(
+ path: str,
+ revision: Optional[Union[str, Version]] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ data_dir: Optional[str] = None,
+ data_files: Optional[Union[dict, list, str, DataFilesDict]] = None,
+ cache_dir: Optional[str] = None,
+ **download_kwargs,
+) -> DatasetModule:
+ """
+ Download/extract/cache a dataset module.
+
+ Dataset codes are cached inside the dynamic modules cache to allow easy import (avoid ugly sys.path tweaks).
+
+ Args:
+
+ path (str): Path or name of the dataset.
+ Depending on ``path``, the dataset builder that is used comes from one of the generic dataset builders (JSON, CSV, Parquet, text etc.).
+
+ For local datasets:
+
+ - if ``path`` is a local directory (containing data files only)
+ -> load a generic dataset builder (csv, json, text etc.) based on the content of the directory
+ e.g. ``'./path/to/directory/with/my/csv/data'``.
+
+ For datasets on the Hugging Face Hub (list all available datasets with ``huggingface_hub.list_datasets()``)
+
+ - if ``path`` is a dataset repository on the HF hub (containing data files only)
+ -> load a generic dataset builder (csv, text etc.) based on the content of the repository
+ e.g. ``'username/dataset_name'``, a dataset repository on the HF hub containing your data files.
+
+ For datasets in Storage Buckets on the Hugging Face Hub (list all available buckets with ``huggingface_hub.list_buckets()``)
+
+ - if `path` is a directory within a Storage Bucket on the HF Hub (containing data files only)
+ -> load the dataset from supported files in the directory (csv, json, parquet, etc.)
+ e.g. `'buckets/username/bucket_name/my_dataset'`.
+
+ revision (:class:`~utils.Version` or :obj:`str`, optional): Version of the dataset to load.
+ As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch.
+ You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository.
+ download_config (:class:`DownloadConfig`, optional): Specific download configuration parameters.
+ download_mode (:class:`DownloadMode` or :obj:`str`, default ``REUSE_DATASET_IF_EXISTS``): Download/generate mode.
+ data_dir (:obj:`str`, optional): Directory with the data files. Used only if `data_files` is not specified,
+ in which case it's equal to pass `os.path.join(data_dir, "**")` as `data_files`.
+ data_files (:obj:`Union[Dict, List, str]`, optional): Defining the data_files of the dataset configuration.
+ cache_dir (`str`, *optional*):
+ Directory to read/write data. Defaults to `"~/.cache/huggingface/datasets"`.
+
+
+
+ **download_kwargs (additional keyword arguments): optional attributes for DownloadConfig() which will override
+ the attributes in download_config if supplied.
+
+ Returns:
+ DatasetModule
+ """
+ if download_config is None:
+ download_config = DownloadConfig(**download_kwargs)
+ download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS)
+ download_config.extract_compressed_file = True
+ download_config.force_extract = True
+ download_config.force_download = download_mode == DownloadMode.FORCE_REDOWNLOAD
+
+ filename = list(filter(lambda x: x, path.replace(os.sep, "/").split("/")))[-1]
+ if not filename.endswith(".py"):
+ filename = filename + ".py"
+ combined_path = os.path.join(path, filename)
+
+ # We have several ways to get a dataset builder:
+ #
+ # - if path is the name of a packaged dataset module
+ # -> use the packaged module (json, csv, etc.)
+ #
+ # - if path is a local directory (but no python file)
+ # -> use a packaged module (csv, text etc.) based on content of the directory
+ #
+ # - if path has one "/" and is dataset repository on the HF hub
+ # -> use a packaged module (csv, text etc.) based on content of the repository
+ #
+ # - if path starts with "buckets/" and points to a Storage Bucket on the HF hub
+ # -> use a packaged module (csv, text etc.) based on content of the directory in the bucket
+
+ if path.startswith("hf://datasets/"):
+ path = path[len("hf://datasets/") :]
+ remote_only = True
+ elif path.startswith("hf://buckets/"):
+ path = path[len("hf://") :]
+ remote_only = True
+ else:
+ remote_only = False
+
+ # Try packaged
+ if path in _PACKAGED_DATASETS_MODULES:
+ return PackagedDatasetModuleFactory(
+ path,
+ data_dir=data_dir,
+ data_files=data_files,
+ download_config=download_config,
+ download_mode=download_mode,
+ ).get_module()
+ # Try locally
+ elif path.endswith(filename):
+ raise RuntimeError(f"Dataset scripts are no longer supported, but found {filename}")
+ elif os.path.isfile(combined_path):
+ raise RuntimeError(f"Dataset scripts are no longer supported, but found {filename}")
+ elif os.path.isdir(path) and not remote_only:
+ return LocalDatasetModuleFactory(
+ path, data_dir=data_dir, data_files=data_files, download_mode=download_mode
+ ).get_module()
+ # Try remotely
+ elif path.startswith("buckets/"):
+ if BucketNotFoundError is None:
+ raise ImportError("Loading datasets from buckets requires huggingface_hub>=1.6.0")
+ # We check that the bucket exists, and the directory exists, and authentication in one call
+ api = HfApi(
+ endpoint=config.HF_ENDPOINT,
+ token=download_config.token,
+ library_name="datasets",
+ library_version=__version__,
+ user_agent=get_datasets_user_agent(download_config.user_agent),
+ )
+ _, _namespace, _bucket_name, *_path_segments = path.split("/")
+ bucket_id = _namespace + "/" + _bucket_name
+ prefix = "/".join(s for s in _path_segments if s)
+ try:
+ next(iter(api.list_bucket_tree(bucket_id, prefix)))
+ except (
+ OfflineModeIsEnabled,
+ requests.exceptions.Timeout,
+ requests.exceptions.ConnectionError,
+ httpx.ConnectError,
+ httpx.TimeoutException,
+ ) as e:
+ raise ConnectionError(f"Couldn't reach '{path}' on the Hub ({e.__class__.__name__})") from e
+ except StopIteration as e:
+ raise DatasetNotFoundError(f"Bucket directory at {path} doesn't exist") from e
+ except BucketNotFoundError as e:
+ raise DatasetNotFoundError(f"Bucket '{bucket_id}' doesn't exist on the Hub or cannot be accessed.") from e
+ return HubBucketDatasetModuleFactory(
+ path,
+ data_dir=data_dir,
+ data_files=data_files,
+ download_config=download_config,
+ download_mode=download_mode,
+ ).get_module()
+ elif is_relative_path(path) and path.count("/") <= 1:
+ try:
+ # Get the Dataset Card + get the revision + check authentication all at in one call
+ # We fix the commit_hash in case there are new commits in the meantime
+ api = HfApi(
+ endpoint=config.HF_ENDPOINT,
+ token=download_config.token,
+ library_name="datasets",
+ library_version=__version__,
+ user_agent=get_datasets_user_agent(download_config.user_agent),
+ )
+ try:
+ _raise_if_offline_mode_is_enabled()
+ dataset_readme_path = api.hf_hub_download(
+ repo_id=path,
+ filename=config.REPOCARD_FILENAME,
+ repo_type="dataset",
+ revision=revision,
+ proxies=download_config.proxies,
+ )
+ commit_hash = os.path.basename(os.path.dirname(dataset_readme_path))
+ except LocalEntryNotFoundError as e:
+ if isinstance(
+ e.__cause__,
+ (
+ OfflineModeIsEnabled,
+ requests.exceptions.Timeout,
+ requests.exceptions.ConnectionError,
+ httpx.ConnectError,
+ httpx.TimeoutException,
+ ),
+ ):
+ raise ConnectionError(f"Couldn't reach '{path}' on the Hub ({e.__class__.__name__})") from e
+ else:
+ raise
+ except EntryNotFoundError:
+ commit_hash = api.dataset_info(
+ path,
+ revision=revision,
+ timeout=100.0,
+ ).sha
+ except (
+ OfflineModeIsEnabled,
+ requests.exceptions.Timeout,
+ requests.exceptions.ConnectionError,
+ httpx.ConnectError,
+ httpx.TimeoutException,
+ ) as e:
+ raise ConnectionError(f"Couldn't reach '{path}' on the Hub ({e.__class__.__name__})") from e
+ except GatedRepoError as e:
+ message = f"Dataset '{path}' is a gated dataset on the Hub."
+ if e.response.status_code == 401:
+ message += " You must be authenticated to access it."
+ elif e.response.status_code == 403:
+ message += f" Visit the dataset page at https://huggingface.co/datasets/{path} to ask for access."
+ raise DatasetNotFoundError(message) from e
+ except RepositoryNotFoundError as e:
+ raise DatasetNotFoundError(f"Dataset '{path}' doesn't exist on the Hub or cannot be accessed.") from e
+ try:
+ api.hf_hub_download(
+ repo_id=path,
+ filename=filename,
+ repo_type="dataset",
+ revision=commit_hash,
+ proxies=download_config.proxies,
+ )
+ raise RuntimeError(f"Dataset scripts are no longer supported, but found {filename}")
+ except EntryNotFoundError:
+ # Use the infos from the parquet export except in some cases:
+ if data_dir or data_files or (revision and revision != "main"):
+ use_exported_dataset_infos = False
+ else:
+ use_exported_dataset_infos = True
+ return HubDatasetModuleFactory(
+ path,
+ commit_hash=commit_hash,
+ data_dir=data_dir,
+ data_files=data_files,
+ download_config=download_config,
+ download_mode=download_mode,
+ use_exported_dataset_infos=use_exported_dataset_infos,
+ ).get_module()
+ except GatedRepoError as e:
+ message = f"Dataset '{path}' is a gated dataset on the Hub."
+ if e.response.status_code == 401:
+ message += " You must be authenticated to access it."
+ elif e.response.status_code == 403:
+ message += f" Visit the dataset page at https://huggingface.co/datasets/{path} to ask for access."
+ raise DatasetNotFoundError(message) from e
+ except RevisionNotFoundError as e:
+ raise DatasetNotFoundError(f"Revision '{revision}' doesn't exist for dataset '{path}' on the Hub.") from e
+ except Exception as e1:
+ # All the attempts failed, before raising the error we should check if the module is already cached
+ try:
+ return CachedDatasetModuleFactory(path, cache_dir=cache_dir).get_module()
+ except Exception:
+ # If it's not in the cache, then it doesn't exist.
+ if isinstance(e1, OfflineModeIsEnabled):
+ raise ConnectionError(f"Couldn't reach the Hugging Face Hub for dataset '{path}': {e1}") from None
+ if isinstance(e1, (DataFilesNotFoundError, DatasetNotFoundError, EmptyDatasetError)):
+ raise e1 from None
+ if isinstance(e1, FileNotFoundError):
+ raise FileNotFoundError(
+ f"Couldn't find any data file at {relative_to_absolute_path(path)}. "
+ f"Couldn't find '{path}' on the Hugging Face Hub either: {type(e1).__name__}: {e1}"
+ ) from None
+ raise e1 from None
+ else:
+ raise FileNotFoundError(f"Couldn't find any data file at {relative_to_absolute_path(path)}.")
+
+
+def load_dataset_builder(
+ path: str,
+ name: Optional[str] = None,
+ data_dir: Optional[str] = None,
+ data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None,
+ cache_dir: Optional[str] = None,
+ features: Optional[Features] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ revision: Optional[Union[str, Version]] = None,
+ token: Optional[Union[bool, str]] = None,
+ storage_options: Optional[dict] = None,
+ **config_kwargs,
+) -> DatasetBuilder:
+ """Load a dataset builder which can be used to:
+
+ - Inspect general information that is required to build a dataset (cache directory, config, dataset info, features, data files, etc.)
+ - Download and prepare the dataset as Arrow files in the cache
+ - Get a streaming dataset without downloading or caching anything
+
+ You can find the list of datasets on the [Hub](https://huggingface.co/datasets) or with [`huggingface_hub.list_datasets`].
+
+ A dataset is a directory that contains some data files in generic formats (JSON, CSV, Parquet, etc.) and possibly
+ in a generic structure (Webdataset, ImageFolder, AudioFolder, VideoFolder, MeshFolder, etc.)
+
+ Args:
+
+ path (`str`):
+ Path or name of the dataset.
+
+ - if `path` is a dataset repository on the HF hub (list all available datasets with [`huggingface_hub.list_datasets`])
+ -> load the dataset builder from supported files in the repository (csv, json, parquet, etc.)
+ e.g. `'username/dataset_name'`, a dataset repository on the HF hub containing the data files.
+
+ - if `path` is a directory within a Storage Bucket on the HF Hub (list your buckets with [`huggingface_hub.list_buckets`])
+ -> load the dataset from supported files in the directory (csv, json, parquet, etc.)
+ e.g. `'buckets/username/bucket_name/my_dataset'`.
+
+ - if `path` is a local directory
+ -> load the dataset builder from supported files in the directory (csv, json, parquet, etc.)
+ e.g. `'./path/to/directory/with/my/csv/data'`.
+
+ - if `path` is the name of a dataset builder and `data_files` or `data_dir` is specified
+ (available builders are "json", "csv", "parquet", "arrow", "text", "xml", "webdataset", "imagefolder", "audiofolder", "videofolder", "meshfolder")
+ -> load the dataset builder from the files in `data_files` or `data_dir`
+ e.g. `'parquet'`.
+
+ Use a `hf://` path like `'hf://datasets/username/dataset_name'` to allow remote only.
+ Use an absolute path to allow local only.
+
+ name (`str`, *optional*):
+ Defining the name of the dataset configuration.
+ data_dir (`str`, *optional*):
+ Defining the `data_dir` of the dataset configuration. If specified for the generic builders (csv, text etc.) or the Hub datasets and `data_files` is `None`,
+ the behavior is equal to passing `os.path.join(data_dir, **)` as `data_files` to reference all the files in a directory.
+ data_files (`str` or `Sequence` or `Mapping`, *optional*):
+ Path(s) to source data file(s).
+ cache_dir (`str`, *optional*):
+ Directory to read/write data. Defaults to `"~/.cache/huggingface/datasets"`.
+ features ([`Features`], *optional*):
+ Set the features type to use for this dataset.
+ download_config ([`DownloadConfig`], *optional*):
+ Specific download configuration parameters.
+ download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`):
+ Download/generate mode.
+ revision ([`Version`] or `str`, *optional*):
+ Version of the dataset to load.
+ As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch.
+ You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository.
+ token (`str` or `bool`, *optional*):
+ Optional string or boolean to use as Bearer token for remote files on the Datasets Hub.
+ If `True`, or not specified, will get token from `"~/.huggingface"`.
+ storage_options (`dict`, *optional*, defaults to `None`):
+ **Experimental**. Key/value pairs to be passed on to the dataset file-system backend, if any.
+
+
+
+ **config_kwargs (additional keyword arguments):
+ Keyword arguments to be passed to the [`BuilderConfig`]
+ and used in the [`DatasetBuilder`].
+
+ Returns:
+ [`DatasetBuilder`]
+
+ Example:
+
+ ```py
+ >>> from datasets import load_dataset_builder
+ >>> ds_builder = load_dataset_builder('cornell-movie-review-data/rotten_tomatoes')
+ >>> ds_builder.info.features
+ {'label': ClassLabel(names=['neg', 'pos']),
+ 'text': Value('string')}
+ ```
+ """
+ download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS)
+ if token is not None:
+ download_config = download_config.copy() if download_config else DownloadConfig()
+ download_config.token = token
+ if storage_options is not None:
+ download_config = download_config.copy() if download_config else DownloadConfig()
+ download_config.storage_options.update(storage_options)
+ if features is not None:
+ features = _fix_for_backward_compatible_features(features)
+ dataset_module = dataset_module_factory(
+ path,
+ revision=revision,
+ download_config=download_config,
+ download_mode=download_mode,
+ data_dir=data_dir,
+ data_files=data_files,
+ cache_dir=cache_dir,
+ )
+ # Get dataset builder class
+ builder_kwargs = dataset_module.builder_kwargs
+ data_dir = builder_kwargs.pop("data_dir", data_dir)
+ data_files = builder_kwargs.pop("data_files", data_files)
+ config_name = builder_kwargs.pop(
+ "config_name", name or dataset_module.builder_configs_parameters.default_config_name
+ )
+ dataset_name = builder_kwargs.pop("dataset_name", None)
+ info = dataset_module.dataset_infos.get(config_name) if dataset_module.dataset_infos else None
+
+ if (
+ path in _PACKAGED_DATASETS_MODULES
+ and data_files is None
+ and dataset_module.builder_configs_parameters.builder_configs[0].data_files is None
+ ):
+ error_msg = f"Please specify the data files or data directory to load for the {path} dataset builder."
+ example_extensions = [
+ extension for extension in _EXTENSION_TO_MODULE if _EXTENSION_TO_MODULE[extension] == path
+ ]
+ if example_extensions:
+ error_msg += f'\nFor example `data_files={{"train": "path/to/data/train/*.{example_extensions[0]}"}}`'
+ raise ValueError(error_msg)
+
+ # When users pass config kwargs, they should override module-provided defaults
+ # instead of colliding at constructor call time.
+ builder_kwargs = {key: value for key, value in builder_kwargs.items() if key not in config_kwargs}
+
+ builder_cls = get_dataset_builder_class(dataset_module, dataset_name=dataset_name)
+ # Instantiate the dataset builder
+ builder_instance: DatasetBuilder = builder_cls(
+ cache_dir=cache_dir,
+ dataset_name=dataset_name,
+ config_name=config_name,
+ data_dir=data_dir,
+ data_files=data_files,
+ hash=dataset_module.hash,
+ info=info,
+ features=features,
+ token=token,
+ storage_options=storage_options,
+ **builder_kwargs,
+ **config_kwargs,
+ )
+ builder_instance._use_legacy_cache_dir_if_possible(dataset_module)
+
+ return builder_instance
+
+
+@overload
+def load_dataset(
+ path: str,
+ name: Optional[str] = None,
+ data_dir: Optional[str] = None,
+ data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None,
+ split: None = None,
+ cache_dir: Optional[str] = None,
+ features: Optional[Features] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ verification_mode: Optional[Union[VerificationMode, str]] = None,
+ keep_in_memory: Optional[bool] = None,
+ save_infos: bool = False,
+ revision: Optional[Union[str, Version]] = None,
+ token: Optional[Union[bool, str]] = None,
+ streaming: Literal[False] = False,
+ num_proc: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ **config_kwargs: Any,
+) -> DatasetDict: ...
+
+
+@overload
+def load_dataset(
+ path: str,
+ name: Optional[str] = None,
+ data_dir: Optional[str] = None,
+ data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None,
+ *,
+ split: Union[str, Split, list[str], list[Split]],
+ cache_dir: Optional[str] = None,
+ features: Optional[Features] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ verification_mode: Optional[Union[VerificationMode, str]] = None,
+ keep_in_memory: Optional[bool] = None,
+ save_infos: bool = False,
+ revision: Optional[Union[Version, str]] = None,
+ token: Optional[Union[bool, str]] = None,
+ streaming: Literal[False] = False,
+ num_proc: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ **config_kwargs: Any,
+) -> Dataset: ...
+
+
+@overload
+def load_dataset(
+ path: str,
+ name: Optional[str] = None,
+ data_dir: Optional[str] = None,
+ data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None,
+ split: None = None,
+ cache_dir: Optional[str] = None,
+ features: Optional[Features] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ verification_mode: Optional[Union[VerificationMode, str]] = None,
+ keep_in_memory: Optional[bool] = None,
+ save_infos: bool = False,
+ revision: Optional[Union[Version, str]] = None,
+ token: Optional[Union[bool, str]] = None,
+ *,
+ streaming: Literal[True],
+ num_proc: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ **config_kwargs: Any,
+) -> IterableDatasetDict: ...
+
+
+@overload
+def load_dataset(
+ path: str,
+ name: Optional[str] = None,
+ data_dir: Optional[str] = None,
+ data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None,
+ *,
+ split: Union[str, Split, list[str], list[Split]],
+ cache_dir: Optional[str] = None,
+ features: Optional[Features] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ verification_mode: Optional[Union[VerificationMode, str]] = None,
+ keep_in_memory: Optional[bool] = None,
+ save_infos: bool = False,
+ revision: Optional[Union[Version, str]] = None,
+ token: Optional[Union[bool, str]] = None,
+ streaming: Literal[True],
+ num_proc: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ **config_kwargs: Any,
+) -> IterableDataset: ...
+
+
+def load_dataset(
+ path: str,
+ name: Optional[str] = None,
+ data_dir: Optional[str] = None,
+ data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None,
+ split: Optional[Union[str, Split, list[str], list[Split]]] = None,
+ cache_dir: Optional[str] = None,
+ features: Optional[Features] = None,
+ download_config: Optional[DownloadConfig] = None,
+ download_mode: Optional[Union[DownloadMode, str]] = None,
+ verification_mode: Optional[Union[VerificationMode, str]] = None,
+ keep_in_memory: Optional[bool] = None,
+ save_infos: bool = False,
+ revision: Optional[Union[str, Version]] = None,
+ token: Optional[Union[bool, str]] = None,
+ streaming: bool = False,
+ num_proc: Optional[int] = None,
+ storage_options: Optional[dict] = None,
+ **config_kwargs,
+) -> Union[DatasetDict, Dataset, IterableDatasetDict, IterableDataset]:
+ """Load a dataset from the Hugging Face Hub, or a local dataset.
+
+ You can find the list of datasets on the [Hub](https://huggingface.co/datasets) or with [`huggingface_hub.list_datasets`].
+
+ A dataset is a directory that contains some data files in generic formats (JSON, CSV, Parquet, etc.) and possibly
+ in a generic structure (Webdataset, ImageFolder, AudioFolder, VideoFolder, MeshFolder, etc.)
+
+ This function does the following under the hood:
+
+ 1. Load a dataset builder:
+
+ * Find the most common data format in the dataset and pick its associated builder (JSON, CSV, Parquet, Webdataset, ImageFolder, AudioFolder, MeshFolder, etc.)
+ * Find which file goes into which split (e.g. train/test) based on file and directory names or on the YAML configuration
+ * It is also possible to specify `data_files` manually, and which dataset builder to use (e.g. "parquet").
+
+ 2. Run the dataset builder:
+
+ In the general case:
+
+ * Download the data files from the dataset if they are not already available locally or cached.
+ * Process and cache the dataset in typed Arrow tables for caching.
+
+ Arrow table are arbitrarily long, typed tables which can store nested objects and be mapped to numpy/pandas/python generic types.
+ They can be directly accessed from disk, loaded in RAM or even streamed over the web.
+
+ In the streaming case:
+
+ * Don't download or cache anything. Instead, the dataset is lazily loaded and will be streamed on-the-fly when iterating on it.
+
+ 3. Return a dataset built from the requested splits in `split` (default: all).
+
+ Args:
+
+ path (`str`):
+ Path or name of the dataset.
+
+ - if `path` is a dataset repository on the HF hub (list all available datasets with [`huggingface_hub.list_datasets`])
+ -> load the dataset from supported files in the repository (csv, json, parquet, etc.)
+ e.g. `'username/dataset_name'`, a dataset repository on the HF hub containing the data files.
+
+ - if `path` is a directory within a Storage Bucket on the HF Hub (list your buckets with [`huggingface_hub.list_buckets`])
+ -> load the dataset from supported files in the directory (csv, json, parquet, etc.)
+ e.g. `'buckets/username/bucket_name/my_dataset'`.
+
+ - if `path` is a local directory
+ -> load the dataset from supported files in the directory (csv, json, parquet, etc.)
+ e.g. `'./path/to/directory/with/my/csv/data'`.
+
+ - if `path` is the name of a dataset builder and `data_files` or `data_dir` is specified
+ (available builders are "json", "csv", "parquet", "arrow", "text", "xml", "webdataset", "imagefolder", "audiofolder", "videofolder", "meshfolder")
+ -> load the dataset from the files in `data_files` or `data_dir`
+ e.g. `'parquet'`.
+
+ Use a `hf://` path like `'hf://datasets/username/dataset_name'` to allow remote only.
+ Use an absolute path to allow local only.
+
+ name (`str`, *optional*):
+ Defining the name of the dataset configuration.
+ data_dir (`str`, *optional*):
+ Defining the `data_dir` of the dataset configuration. If specified for the generic builders (csv, text etc.) or the Hub datasets and `data_files` is `None`,
+ the behavior is equal to passing `os.path.join(data_dir, **)` as `data_files` to reference all the files in a directory.
+ data_files (`str` or `Sequence` or `Mapping`, *optional*):
+ Path(s) to source data file(s).
+ split (`Split` or `str`):
+ Which split of the data to load.
+ If `None`, will return a `dict` with all splits (typically `datasets.Split.TRAIN` and `datasets.Split.TEST`).
+ If given, will return a single Dataset.
+ Splits can be combined and specified like in tensorflow-datasets.
+ cache_dir (`str`, *optional*):
+ Directory to read/write data. Defaults to `"~/.cache/huggingface/datasets"`.
+ features (`Features`, *optional*):
+ Set the features type to use for this dataset.
+ download_config ([`DownloadConfig`], *optional*):
+ Specific download configuration parameters.
+ download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`):
+ Download/generate mode.
+ verification_mode ([`VerificationMode`] or `str`, defaults to `BASIC_CHECKS`):
+ Verification mode determining the checks to run on the downloaded/processed dataset information (checksums/size/splits/...).
+
+
+ keep_in_memory (`bool`, defaults to `None`):
+ Whether to copy the dataset in-memory. If `None`, the dataset
+ will not be copied in-memory unless explicitly enabled by setting `datasets.config.IN_MEMORY_MAX_SIZE` to
+ nonzero. See more details in the [improve performance](../cache#improve-performance) section.
+ revision ([`Version`] or `str`, *optional*):
+ Version of the dataset to load.
+ As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch.
+ You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository.
+ token (`str` or `bool`, *optional*):
+ Optional string or boolean to use as Bearer token for remote files on the Datasets Hub.
+ If `True`, or not specified, will get token from `"~/.huggingface"`.
+ streaming (`bool`, defaults to `False`):
+ If set to `True`, don't download the data files. Instead, it streams the data progressively while
+ iterating on the dataset. An [`IterableDataset`] or [`IterableDatasetDict`] is returned instead in this case.
+
+ Note that streaming works for datasets that use data formats that support being iterated over like txt, csv, jsonl for example.
+ Json files may be downloaded completely. Also streaming from remote zip or gzip files is supported but other compressed formats
+ like rar and xz are not yet supported. The tgz format doesn't allow streaming.
+ num_proc (`int`, *optional*, defaults to `None`):
+ Number of processes when downloading and generating the dataset locally.
+ Multiprocessing is disabled by default.
+
+
+ storage_options (`dict`, *optional*, defaults to `None`):
+ **Experimental**. Key/value pairs to be passed on to the dataset file-system backend, if any.
+
+
+ **config_kwargs (additional keyword arguments):
+ Keyword arguments to be passed to the `BuilderConfig`
+ and used in the [`DatasetBuilder`].
+
+ Returns:
+ [`Dataset`] or [`DatasetDict`]:
+ - if `split` is not `None`: the dataset requested,
+ - if `split` is `None`, a [`~datasets.DatasetDict`] with each split.
+
+ or [`IterableDataset`] or [`IterableDatasetDict`]: if `streaming=True`
+
+ - if `split` is not `None`, the dataset is requested
+ - if `split` is `None`, a [`~datasets.streaming.IterableDatasetDict`] with each split.
+
+ Example:
+
+ Load a dataset from the Hugging Face Hub:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset('cornell-movie-review-data/rotten_tomatoes', split='train')
+
+ # Load a subset or dataset configuration (here 'sst2')
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset('nyu-mll/glue', 'sst2', split='train')
+
+ # Manual mapping of data files to splits
+ >>> data_files = {'train': 'train.csv', 'test': 'test.csv'}
+ >>> ds = load_dataset('namespace/your_dataset_name', data_files=data_files)
+
+ # Manual selection of a directory to load
+ >>> ds = load_dataset('namespace/your_dataset_name', data_dir='folder_name')
+ ```
+
+ Load a dataset from a Storage Bucket on the Hugging Face Hub:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset('buckets/username/bucket_name/rotten_tomatoes', split='train')
+ ```
+
+ Load a local dataset:
+
+ ```py
+ # Load a CSV file
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset('csv', data_files='path/to/local/my_dataset.csv')
+
+ # Load a JSON file
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset('json', data_files='path/to/local/my_dataset.json')
+ ```
+
+ Load an [`~datasets.IterableDataset`]:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset('cornell-movie-review-data/rotten_tomatoes', split='train', streaming=True)
+ ```
+
+ Load an image dataset with the `ImageFolder` dataset builder:
+
+ ```py
+ >>> from datasets import load_dataset
+ >>> ds = load_dataset('imagefolder', data_dir='/path/to/images', split='train')
+ ```
+ """
+ if "trust_remote_code" in config_kwargs:
+ if config_kwargs.pop("trust_remote_code"):
+ logger.error(
+ "`trust_remote_code` is not supported anymore.\n"
+ f"Please check that the Hugging Face dataset '{path}' isn't based on a loading script and remove `trust_remote_code`.\n"
+ "If the dataset is based on a loading script, please ask the dataset author to remove it and convert it to a standard format like Parquet."
+ )
+ if data_files is not None and not data_files:
+ raise ValueError(f"Empty 'data_files': '{data_files}'. It should be either non-empty or None (default).")
+ if Path(path, config.DATASET_STATE_JSON_FILENAME).exists():
+ raise ValueError(
+ "You are trying to load a dataset that was saved using `save_to_disk`. "
+ "Please use `load_from_disk` instead."
+ )
+
+ if streaming and num_proc is not None:
+ raise NotImplementedError(
+ "Loading a streaming dataset in parallel with `num_proc` is not implemented. "
+ "To parallelize streaming, you can wrap the dataset with a PyTorch DataLoader using `num_workers` > 1 instead."
+ )
+
+ download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS)
+ verification_mode = VerificationMode(
+ (verification_mode or VerificationMode.BASIC_CHECKS) if not save_infos else VerificationMode.ALL_CHECKS
+ )
+
+ # Create a dataset builder
+ builder_instance = load_dataset_builder(
+ path=path,
+ name=name,
+ data_dir=data_dir,
+ data_files=data_files,
+ cache_dir=cache_dir,
+ features=features,
+ download_config=download_config,
+ download_mode=download_mode,
+ revision=revision,
+ token=token,
+ storage_options=storage_options,
+ **config_kwargs,
+ )
+
+ # Return iterable dataset in case of streaming
+ if streaming:
+ return builder_instance.as_streaming_dataset(split=split)
+
+ # Download and prepare data
+ builder_instance.download_and_prepare(
+ download_config=download_config,
+ download_mode=download_mode,
+ verification_mode=verification_mode,
+ num_proc=num_proc,
+ storage_options=storage_options,
+ )
+
+ # Build dataset for splits
+ keep_in_memory = (
+ keep_in_memory if keep_in_memory is not None else is_small_dataset(builder_instance.info.dataset_size)
+ )
+ ds = builder_instance.as_dataset(split=split, in_memory=keep_in_memory)
+
+ return ds
+
+
+def load_from_disk(
+ dataset_path: PathLike, keep_in_memory: Optional[bool] = None, storage_options: Optional[dict] = None
+) -> Union[Dataset, DatasetDict]:
+ """
+ Loads a dataset that was previously saved using [`~Dataset.save_to_disk`] from a dataset directory, or
+ from a filesystem using any implementation of `fsspec.spec.AbstractFileSystem`.
+
+ Args:
+ dataset_path (`path-like`):
+ Path (e.g. `"dataset/train"`) or remote URI (e.g. `"s3://my-bucket/dataset/train"`)
+ of the [`Dataset`] or [`DatasetDict`] directory where the dataset/dataset-dict will be
+ loaded from.
+ keep_in_memory (`bool`, defaults to `None`):
+ Whether to copy the dataset in-memory. If `None`, the dataset
+ will not be copied in-memory unless explicitly enabled by setting `datasets.config.IN_MEMORY_MAX_SIZE` to
+ nonzero. See more details in the [improve performance](../cache#improve-performance) section.
+
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+
+
+
+ Returns:
+ [`Dataset`] or [`DatasetDict`]:
+ - If `dataset_path` is a path of a dataset directory: the dataset requested.
+ - If `dataset_path` is a path of a dataset dict directory, a [`DatasetDict`] with each split.
+
+ Example:
+
+ ```py
+ >>> from datasets import load_from_disk
+ >>> ds = load_from_disk('path/to/dataset/directory')
+ ```
+ """
+ fs: fsspec.AbstractFileSystem
+ fs, *_ = url_to_fs(dataset_path, **(storage_options or {}))
+ if not fs.exists(dataset_path):
+ raise FileNotFoundError(f"Directory {dataset_path} not found")
+ if fs.isfile(posixpath.join(dataset_path, config.DATASET_INFO_FILENAME)) and fs.isfile(
+ posixpath.join(dataset_path, config.DATASET_STATE_JSON_FILENAME)
+ ):
+ return Dataset.load_from_disk(dataset_path, keep_in_memory=keep_in_memory, storage_options=storage_options)
+ elif fs.isfile(posixpath.join(dataset_path, config.DATASETDICT_JSON_FILENAME)):
+ return DatasetDict.load_from_disk(dataset_path, keep_in_memory=keep_in_memory, storage_options=storage_options)
+ else:
+ raise FileNotFoundError(
+ f"Directory {dataset_path} is neither a `Dataset` directory nor a `DatasetDict` directory."
+ )
diff --git a/venv/Lib/site-packages/datasets/naming.py b/venv/Lib/site-packages/datasets/naming.py
new file mode 100644
index 0000000000000000000000000000000000000000..29216bbd7de3b27993f821a1d26604ab40337f3c
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/naming.py
@@ -0,0 +1,84 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Utilities for file names."""
+
+import itertools
+import os
+import re
+
+
+_uppercase_uppercase_re = re.compile(r"([A-Z]+)([A-Z][a-z])")
+_lowercase_uppercase_re = re.compile(r"([a-z\d])([A-Z])")
+
+_single_underscore_re = re.compile(r"(?:/\|?*"
+
+
+def camelcase_to_snakecase(name):
+ """Convert camel-case string to snake-case."""
+ name = _uppercase_uppercase_re.sub(r"\1_\2", name)
+ name = _lowercase_uppercase_re.sub(r"\1_\2", name)
+ return name.lower()
+
+
+def snakecase_to_camelcase(name):
+ """Convert snake-case string to camel-case string."""
+ name = _single_underscore_re.split(name)
+ name = [_multiple_underscores_re.split(n) for n in name]
+ return "".join(n.capitalize() for n in itertools.chain.from_iterable(name) if n != "")
+
+
+def filename_prefix_for_name(name):
+ if os.path.basename(name) != name:
+ raise ValueError(f"Should be a dataset name, not a path: {name}")
+ return camelcase_to_snakecase(name)
+
+
+def filename_prefix_for_split(name, split):
+ if os.path.basename(name) != name:
+ raise ValueError(f"Should be a dataset name, not a path: {name}")
+ if not re.match(_split_re, split):
+ raise ValueError(f"Split name should match '{_split_re}'' but got '{split}'.")
+ return f"{filename_prefix_for_name(name)}-{split}"
+
+
+def filepattern_for_dataset_split(dataset_name, split, data_dir, filetype_suffix=None):
+ prefix = filename_prefix_for_split(dataset_name, split)
+ if filetype_suffix:
+ prefix += f".{filetype_suffix}"
+ filepath = os.path.join(data_dir, prefix)
+ return f"{filepath}*"
+
+
+def filenames_for_dataset_split(path, dataset_name, split, filetype_suffix=None, shard_lengths=None):
+ prefix = filename_prefix_for_split(dataset_name, split)
+ prefix = os.path.join(path, prefix)
+
+ if shard_lengths and len(shard_lengths) > 1:
+ num_shards = len(shard_lengths)
+ filenames = [f"{prefix}-{shard_id:05d}-of-{num_shards:05d}" for shard_id in range(num_shards)]
+ if filetype_suffix:
+ filenames = [filename + f".{filetype_suffix}" for filename in filenames]
+ return filenames
+ else:
+ filename = prefix
+ if filetype_suffix:
+ filename += f".{filetype_suffix}"
+ return [filename]
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/__init__.py b/venv/Lib/site-packages/datasets/packaged_modules/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..0de63b9c785c74282857548c514156a6ff091428
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/__init__.py
@@ -0,0 +1,148 @@
+import inspect
+import re
+from typing import Dict, List, Tuple
+
+from huggingface_hub.utils import insecure_hashlib
+
+from .arrow import arrow
+from .audiofolder import audiofolder
+from .cache import cache
+from .conll import conll
+from .csv import csv
+from .eval import eval
+from .hdf5 import hdf5
+from .iceberg import iceberg
+from .imagefolder import imagefolder
+from .json import json
+from .lance import lance
+from .meshfolder import meshfolder
+from .niftifolder import niftifolder
+from .pandas import pandas
+from .parquet import parquet
+from .pdffolder import pdffolder
+from .sql import sql
+from .text import text
+from .tsfile import tsfile
+from .videofolder import videofolder
+from .webdataset import webdataset
+from .xml import xml
+
+
+def _hash_python_lines(lines: list[str]) -> str:
+ filtered_lines = []
+ for line in lines:
+ line = re.sub(r"#.*", "", line) # remove comments
+ if line:
+ filtered_lines.append(line)
+ full_str = "\n".join(filtered_lines)
+
+ # Make a hash from all this code
+ full_bytes = full_str.encode("utf-8")
+ return insecure_hashlib.sha256(full_bytes).hexdigest()
+
+
+# get importable module names and hash for caching
+_PACKAGED_DATASETS_MODULES = {
+ "csv": (csv.__name__, _hash_python_lines(inspect.getsource(csv).splitlines())),
+ "json": (json.__name__, _hash_python_lines(inspect.getsource(json).splitlines())),
+ "pandas": (pandas.__name__, _hash_python_lines(inspect.getsource(pandas).splitlines())),
+ "parquet": (parquet.__name__, _hash_python_lines(inspect.getsource(parquet).splitlines())),
+ "arrow": (arrow.__name__, _hash_python_lines(inspect.getsource(arrow).splitlines())),
+ "text": (text.__name__, _hash_python_lines(inspect.getsource(text).splitlines())),
+ "conll": (conll.__name__, _hash_python_lines(inspect.getsource(conll).splitlines())),
+ "imagefolder": (imagefolder.__name__, _hash_python_lines(inspect.getsource(imagefolder).splitlines())),
+ "audiofolder": (audiofolder.__name__, _hash_python_lines(inspect.getsource(audiofolder).splitlines())),
+ "videofolder": (videofolder.__name__, _hash_python_lines(inspect.getsource(videofolder).splitlines())),
+ "meshfolder": (meshfolder.__name__, _hash_python_lines(inspect.getsource(meshfolder).splitlines())),
+ "pdffolder": (pdffolder.__name__, _hash_python_lines(inspect.getsource(pdffolder).splitlines())),
+ "niftifolder": (niftifolder.__name__, _hash_python_lines(inspect.getsource(niftifolder).splitlines())),
+ "webdataset": (webdataset.__name__, _hash_python_lines(inspect.getsource(webdataset).splitlines())),
+ "xml": (xml.__name__, _hash_python_lines(inspect.getsource(xml).splitlines())),
+ "hdf5": (hdf5.__name__, _hash_python_lines(inspect.getsource(hdf5).splitlines())),
+ "eval": (eval.__name__, _hash_python_lines(inspect.getsource(eval).splitlines())),
+ "lance": (lance.__name__, _hash_python_lines(inspect.getsource(lance).splitlines())),
+ "tsfile": (tsfile.__name__, _hash_python_lines(inspect.getsource(tsfile).splitlines())),
+ "iceberg": (iceberg.__name__, _hash_python_lines(inspect.getsource(iceberg).splitlines())),
+}
+
+# get importable module names and hash for caching
+_PACKAGED_DATASETS_MODULES_2_15_HASHES = {
+ "csv": "eea64c71ca8b46dd3f537ed218fc9bf495d5707789152eb2764f5c78fa66d59d",
+ "json": "8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96",
+ "pandas": "3ac4ffc4563c796122ef66899b9485a3f1a977553e2d2a8a318c72b8cc6f2202",
+ "parquet": "ca31c69184d9832faed373922c2acccec0b13a0bb5bbbe19371385c3ff26f1d1",
+ "arrow": "74f69db2c14c2860059d39860b1f400a03d11bf7fb5a8258ca38c501c878c137",
+ "text": "c4a140d10f020282918b5dd1b8a49f0104729c6177f60a6b49ec2a365ec69f34",
+ "imagefolder": "7b7ce5247a942be131d49ad4f3de5866083399a0f250901bd8dc202f8c5f7ce5",
+ "audiofolder": "d3c1655c66c8f72e4efb5c79e952975fa6e2ce538473a6890241ddbddee9071c",
+}
+
+# Used to infer the module to use based on the data files extensions
+_EXTENSION_TO_MODULE: dict[str, tuple[str, dict]] = {
+ ".csv": ("csv", {}),
+ ".tsv": ("csv", {"sep": "\t"}),
+ ".json": ("json", {}),
+ ".jsonl": ("json", {}),
+ # ndjson is no longer maintained (see: https://github.com/ndjson/ndjson-spec/issues/35#issuecomment-1285673417)
+ ".ndjson": ("json", {}),
+ ".parquet": ("parquet", {}),
+ ".geoparquet": ("parquet", {}),
+ ".gpq": ("parquet", {}),
+ ".arrow": ("arrow", {}),
+ ".txt": ("text", {}),
+ ".conll": ("conll", {}),
+ ".conllu": ("conll", {"comment_prefix": "#"}),
+ ".tar": ("webdataset", {}),
+ ".xml": ("xml", {}),
+ ".hdf5": ("hdf5", {}),
+ ".h5": ("hdf5", {}),
+ ".eval": ("eval", {}),
+ ".lance": ("lance", {}),
+ ".tsfile": ("tsfile", {}),
+}
+_EXTENSION_TO_MODULE.update({ext: ("imagefolder", {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
+_EXTENSION_TO_MODULE.update({ext.upper(): ("imagefolder", {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
+_EXTENSION_TO_MODULE.update({ext: ("audiofolder", {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
+_EXTENSION_TO_MODULE.update({ext.upper(): ("audiofolder", {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
+_EXTENSION_TO_MODULE.update({ext: ("videofolder", {}) for ext in videofolder.VideoFolder.EXTENSIONS})
+_EXTENSION_TO_MODULE.update({ext.upper(): ("videofolder", {}) for ext in videofolder.VideoFolder.EXTENSIONS})
+_EXTENSION_TO_MODULE.update({ext: ("meshfolder", {}) for ext in meshfolder.MeshFolder.EXTENSIONS})
+_EXTENSION_TO_MODULE.update({ext.upper(): ("meshfolder", {}) for ext in meshfolder.MeshFolder.EXTENSIONS})
+_EXTENSION_TO_MODULE.update({ext: ("pdffolder", {}) for ext in pdffolder.PdfFolder.EXTENSIONS})
+_EXTENSION_TO_MODULE.update({ext.upper(): ("pdffolder", {}) for ext in pdffolder.PdfFolder.EXTENSIONS})
+_EXTENSION_TO_MODULE.update({ext: ("niftifolder", {}) for ext in niftifolder.NiftiFolder.EXTENSIONS})
+_EXTENSION_TO_MODULE.update({ext.upper(): ("niftifolder", {}) for ext in niftifolder.NiftiFolder.EXTENSIONS})
+
+# Used to filter data files based on extensions given a module name
+_MODULE_TO_EXTENSIONS: dict[str, list[str]] = {}
+for _ext, (_module, _) in _EXTENSION_TO_MODULE.items():
+ _MODULE_TO_EXTENSIONS.setdefault(_module, []).append(_ext)
+
+for _module in _MODULE_TO_EXTENSIONS:
+ _MODULE_TO_EXTENSIONS[_module].append(".zip")
+
+# Used to filter data files based on file names
+_MODULE_TO_METADATA_FILE_NAMES: Dict[str, List[str]] = {}
+for _module in _MODULE_TO_EXTENSIONS:
+ _MODULE_TO_METADATA_FILE_NAMES[_module] = []
+_MODULE_TO_METADATA_FILE_NAMES["imagefolder"] = imagefolder.ImageFolder.METADATA_FILENAMES
+_MODULE_TO_METADATA_FILE_NAMES["audiofolder"] = imagefolder.ImageFolder.METADATA_FILENAMES
+_MODULE_TO_METADATA_FILE_NAMES["videofolder"] = imagefolder.ImageFolder.METADATA_FILENAMES
+_MODULE_TO_METADATA_FILE_NAMES["meshfolder"] = meshfolder.MeshFolder.METADATA_FILENAMES
+_MODULE_TO_METADATA_FILE_NAMES["pdffolder"] = imagefolder.ImageFolder.METADATA_FILENAMES
+_MODULE_TO_METADATA_FILE_NAMES["niftifolder"] = imagefolder.ImageFolder.METADATA_FILENAMES
+_MODULE_TO_METADATA_FILE_NAMES["lance"] = lance.Lance.METADATA_FILE_NAMES
+
+_MODULE_TO_METADATA_EXTENSIONS: Dict[str, List[str]] = {}
+for _module in _MODULE_TO_EXTENSIONS:
+ _MODULE_TO_METADATA_EXTENSIONS[_module] = []
+_MODULE_TO_METADATA_EXTENSIONS["lance"] = lance.Lance.METADATA_EXTENSIONS
+
+# Total
+
+_ALL_EXTENSIONS = list(_EXTENSION_TO_MODULE.keys()) + [".zip"]
+_ALL_METADATA_EXTENSIONS = sorted({_ext for _exts in _MODULE_TO_METADATA_EXTENSIONS.values() for _ext in _exts})
+_ALL_ALLOWED_EXTENSIONS = _ALL_EXTENSIONS + _ALL_METADATA_EXTENSIONS
+_ALL_METADATA_FILENAMES = sorted(
+ {file_name for file_names in _MODULE_TO_METADATA_FILE_NAMES.values() for file_name in file_names}
+)
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3f61ed18936aa490db8ab7dcaa3366146d9624ec
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/arrow/__init__.py b/venv/Lib/site-packages/datasets/packaged_modules/arrow/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/arrow/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/arrow/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b79f0c33099927c16291e727f5cb0807eec7df65
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/arrow/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/arrow/__pycache__/arrow.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/arrow/__pycache__/arrow.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..763bebb108f04a236c69e448cc897dcf83bdf3a2
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/arrow/__pycache__/arrow.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/arrow/arrow.py b/venv/Lib/site-packages/datasets/packaged_modules/arrow/arrow.py
new file mode 100644
index 0000000000000000000000000000000000000000..7a655d98831a55cc2a428ec74c6d46822f8cae98
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/arrow/arrow.py
@@ -0,0 +1,77 @@
+from dataclasses import dataclass
+from typing import Optional
+
+import pyarrow as pa
+
+import datasets
+from datasets.builder import Key
+from datasets.table import table_cast
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+@dataclass
+class ArrowConfig(datasets.BuilderConfig):
+ """BuilderConfig for Arrow."""
+
+ features: Optional[datasets.Features] = None
+
+ def __post_init__(self):
+ super().__post_init__()
+
+
+class Arrow(datasets.ArrowBasedBuilder):
+ BUILDER_CONFIG_CLASS = ArrowConfig
+
+ def _info(self):
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager):
+ """We handle string, list and dicts in datafiles"""
+ if not self.config.data_files:
+ raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+ data_files = dl_manager.download(self.config.data_files)
+ splits = []
+ for split_name, files in data_files.items():
+ # Infer features if they are stored in the arrow schema
+ if self.info.features is None:
+ for file in files:
+ with open(file, "rb") as f:
+ try:
+ reader = pa.ipc.open_stream(f)
+ except (OSError, pa.lib.ArrowInvalid):
+ reader = pa.ipc.open_file(f)
+ self.info.features = datasets.Features.from_arrow_schema(reader.schema)
+ break
+ splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files}))
+ return splits
+
+ def _cast_table(self, pa_table: pa.Table) -> pa.Table:
+ if self.info.features is not None:
+ # more expensive cast to support nested features with keys in a different order
+ # allows str <-> int/float or str to Audio for example
+ pa_table = table_cast(pa_table, self.info.features.arrow_schema)
+ return pa_table
+
+ def _generate_shards(self, files):
+ yield from files
+
+ def _generate_tables(self, files):
+ for file_idx, file in enumerate(files):
+ with open(file, "rb") as f:
+ try:
+ try:
+ batches = pa.ipc.open_stream(f)
+ except (OSError, pa.lib.ArrowInvalid):
+ reader = pa.ipc.open_file(f)
+ batches = (reader.get_batch(i) for i in range(reader.num_record_batches))
+ for batch_idx, record_batch in enumerate(batches):
+ pa_table = pa.Table.from_batches([record_batch])
+ # Uncomment for debugging (will print the Arrow table size and elements)
+ # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
+ # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
+ yield Key(file_idx, batch_idx), self._cast_table(pa_table)
+ except ValueError as e:
+ logger.error(f"Failed to read file '{file}' with error {type(e)}: {e}")
+ raise
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/audiofolder/__init__.py b/venv/Lib/site-packages/datasets/packaged_modules/audiofolder/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/audiofolder/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/audiofolder/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f87d89dae829fee92620f3b6944ed86fb358ac3b
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/audiofolder/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/audiofolder/__pycache__/audiofolder.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/audiofolder/__pycache__/audiofolder.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..bc6268b184accbe615d810ed30bc58ba7286cff9
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/audiofolder/__pycache__/audiofolder.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/audiofolder/audiofolder.py b/venv/Lib/site-packages/datasets/packaged_modules/audiofolder/audiofolder.py
new file mode 100644
index 0000000000000000000000000000000000000000..9e71a6bbbae8d138c8bf637846cdbebc1a5dc8f8
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/audiofolder/audiofolder.py
@@ -0,0 +1,86 @@
+import datasets
+
+from ..folder_based_builder import folder_based_builder
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+class AudioFolderConfig(folder_based_builder.FolderBasedBuilderConfig):
+ """Builder Config for AudioFolder."""
+
+ drop_labels: bool = None
+ drop_metadata: bool = None
+
+ def __post_init__(self):
+ super().__post_init__()
+
+
+class AudioFolder(folder_based_builder.FolderBasedBuilder):
+ BASE_FEATURE = datasets.Audio
+ BASE_COLUMN_NAME = "audio"
+ BUILDER_CONFIG_CLASS = AudioFolderConfig
+ EXTENSIONS: list[str] # definition at the bottom of the script
+
+
+# Obtained with:
+# ```
+# import soundfile as sf
+#
+# AUDIO_EXTENSIONS = [f".{format.lower()}" for format in sf.available_formats().keys()]
+#
+# # .opus decoding is supported if libsndfile >= 1.0.31:
+# AUDIO_EXTENSIONS.extend([".opus"])
+# ```
+# We intentionally did not run this code on launch because:
+# (1) Soundfile was an optional dependency, so importing it in global namespace is not allowed
+# (2) To ensure the list of supported extensions is deterministic
+# (3) We use TorchCodec now anyways instead of Soundfile
+AUDIO_EXTENSIONS = [
+ ".aiff",
+ ".au",
+ ".avr",
+ ".caf",
+ ".flac",
+ ".htk",
+ ".svx",
+ ".mat4",
+ ".mat5",
+ ".mpc2k",
+ ".ogg",
+ ".paf",
+ ".pvf",
+ ".raw",
+ ".rf64",
+ ".sd2",
+ ".sds",
+ ".ircam",
+ ".voc",
+ ".w64",
+ ".wav",
+ ".nist",
+ ".wavex",
+ ".wve",
+ ".xi",
+ ".mp3",
+ ".opus",
+ ".3gp",
+ ".3g2",
+ ".avi",
+ ".asf",
+ ".flv",
+ ".mp4",
+ ".mov",
+ ".m4v",
+ ".mkv",
+ ".mpg",
+ ".webm",
+ ".f4v",
+ ".wmv",
+ ".wma",
+ ".ogg",
+ ".ogm",
+ ".mxf",
+ ".nut",
+]
+AudioFolder.EXTENSIONS = AUDIO_EXTENSIONS
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/cache/__init__.py b/venv/Lib/site-packages/datasets/packaged_modules/cache/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/cache/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/cache/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b47581dce4c057a67cdefd621cdd76f16425ca2d
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/cache/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/cache/__pycache__/cache.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/cache/__pycache__/cache.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..2a9879f61fe5296cb2d95795cfe457f401c09e0a
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/cache/__pycache__/cache.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/cache/cache.py b/venv/Lib/site-packages/datasets/packaged_modules/cache/cache.py
new file mode 100644
index 0000000000000000000000000000000000000000..a5d9f3585dec8b50c0d3422af84078118cb4332f
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/cache/cache.py
@@ -0,0 +1,195 @@
+import glob
+import json
+import os
+import shutil
+import time
+from pathlib import Path
+from typing import Optional, Union
+
+import pyarrow as pa
+
+import datasets
+import datasets.config
+import datasets.data_files
+from datasets.builder import Key
+from datasets.naming import camelcase_to_snakecase, filenames_for_dataset_split
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+def _get_modification_time(cached_directory_path):
+ return (Path(cached_directory_path)).stat().st_mtime
+
+
+def _find_hash_in_cache(
+ dataset_name: str,
+ config_name: Optional[str],
+ cache_dir: Optional[str],
+ config_kwargs: dict,
+ custom_features: Optional[datasets.Features],
+) -> tuple[str, str, str]:
+ if config_name or config_kwargs or custom_features:
+ config_id = datasets.BuilderConfig(config_name or "default").create_config_id(
+ config_kwargs=config_kwargs, custom_features=custom_features
+ )
+ else:
+ config_id = None
+ cache_dir = os.path.expanduser(str(cache_dir or datasets.config.HF_DATASETS_CACHE))
+ namespace_and_dataset_name = dataset_name.split("/")
+ namespace_and_dataset_name[-1] = camelcase_to_snakecase(namespace_and_dataset_name[-1])
+ cached_relative_path = "___".join(namespace_and_dataset_name)
+ cached_datasets_directory_path_root = os.path.join(cache_dir, cached_relative_path)
+ cached_directory_paths = [
+ cached_directory_path
+ for cached_directory_path in glob.glob(
+ os.path.join(cached_datasets_directory_path_root, config_id or "*", "*", "*")
+ )
+ if os.path.isdir(cached_directory_path)
+ and (
+ config_kwargs
+ or custom_features
+ or json.loads(Path(cached_directory_path, "dataset_info.json").read_text(encoding="utf-8"))["config_name"]
+ == Path(cached_directory_path).parts[-3] # no extra params => config_id == config_name
+ )
+ ]
+ if not cached_directory_paths:
+ cached_directory_paths = [
+ cached_directory_path
+ for cached_directory_path in glob.glob(os.path.join(cached_datasets_directory_path_root, "*", "*", "*"))
+ if os.path.isdir(cached_directory_path)
+ ]
+ available_configs = sorted(
+ {Path(cached_directory_path).parts[-3] for cached_directory_path in cached_directory_paths}
+ )
+ raise ValueError(
+ f"Couldn't find cache for {dataset_name}"
+ + (f" for config '{config_id}'" if config_id else "")
+ + (f"\nAvailable configs in the cache: {available_configs}" if available_configs else "")
+ )
+ # get most recent
+ cached_directory_path = Path(sorted(cached_directory_paths, key=_get_modification_time)[-1])
+ version, hash = cached_directory_path.parts[-2:]
+ other_configs = [
+ Path(_cached_directory_path).parts[-3]
+ for _cached_directory_path in glob.glob(os.path.join(cached_datasets_directory_path_root, "*", version, hash))
+ if os.path.isdir(_cached_directory_path)
+ and (
+ config_kwargs
+ or custom_features
+ or json.loads(Path(_cached_directory_path, "dataset_info.json").read_text(encoding="utf-8"))["config_name"]
+ == Path(_cached_directory_path).parts[-3] # no extra params => config_id == config_name
+ )
+ ]
+ if not config_id and len(other_configs) > 1:
+ raise ValueError(
+ f"There are multiple '{dataset_name}' configurations in the cache: {', '.join(other_configs)}"
+ f"\nPlease specify which configuration to reload from the cache, e.g."
+ f"\n\tload_dataset('{dataset_name}', '{other_configs[0]}')"
+ )
+ config_name = cached_directory_path.parts[-3]
+ warning_msg = (
+ f"Found the latest cached dataset configuration '{config_name}' at {cached_directory_path} "
+ f"(last modified on {time.ctime(_get_modification_time(cached_directory_path))})."
+ )
+ logger.warning(warning_msg)
+ return config_name, version, hash
+
+
+class Cache(datasets.ArrowBasedBuilder):
+ def __init__(
+ self,
+ cache_dir: Optional[str] = None,
+ dataset_name: Optional[str] = None,
+ config_name: Optional[str] = None,
+ version: Optional[str] = "0.0.0",
+ hash: Optional[str] = None,
+ base_path: Optional[str] = None,
+ info: Optional[datasets.DatasetInfo] = None,
+ features: Optional[datasets.Features] = None,
+ token: Optional[Union[bool, str]] = None,
+ repo_id: Optional[str] = None,
+ data_files: Optional[Union[str, list, dict, datasets.data_files.DataFilesDict]] = None,
+ data_dir: Optional[str] = None,
+ storage_options: Optional[dict] = None,
+ writer_batch_size: Optional[int] = None,
+ **config_kwargs,
+ ):
+ if repo_id is None and dataset_name is None:
+ raise ValueError("repo_id or dataset_name is required for the Cache dataset builder")
+ if data_files is not None:
+ config_kwargs["data_files"] = data_files
+ if data_dir is not None:
+ config_kwargs["data_dir"] = data_dir
+ if hash == "auto" and version == "auto":
+ config_name, version, hash = _find_hash_in_cache(
+ dataset_name=repo_id or dataset_name,
+ config_name=config_name,
+ cache_dir=cache_dir,
+ config_kwargs=config_kwargs,
+ custom_features=features,
+ )
+ elif hash == "auto" or version == "auto":
+ raise NotImplementedError("Pass both hash='auto' and version='auto' instead")
+ super().__init__(
+ cache_dir=cache_dir,
+ dataset_name=dataset_name,
+ config_name=config_name,
+ version=version,
+ hash=hash,
+ base_path=base_path,
+ info=info,
+ token=token,
+ repo_id=repo_id,
+ storage_options=storage_options,
+ writer_batch_size=writer_batch_size,
+ )
+
+ def _info(self) -> datasets.DatasetInfo:
+ return datasets.DatasetInfo()
+
+ def download_and_prepare(self, output_dir: Optional[str] = None, *args, **kwargs):
+ if not os.path.exists(self.cache_dir):
+ raise ValueError(f"Cache directory for {self.dataset_name} doesn't exist at {self.cache_dir}")
+ if output_dir is not None and output_dir != self.cache_dir:
+ shutil.copytree(self.cache_dir, output_dir)
+
+ def _split_generators(self, dl_manager):
+ # used to stream from cache
+ if isinstance(self.info.splits, datasets.SplitDict):
+ split_infos: list[datasets.SplitInfo] = list(self.info.splits.values())
+ else:
+ raise ValueError(f"Missing splits info for {self.dataset_name} in cache directory {self.cache_dir}")
+ return [
+ datasets.SplitGenerator(
+ name=split_info.name,
+ gen_kwargs={
+ "files": filenames_for_dataset_split(
+ self.cache_dir,
+ dataset_name=self.dataset_name,
+ split=split_info.name,
+ filetype_suffix="arrow",
+ shard_lengths=split_info.shard_lengths,
+ )
+ },
+ )
+ for split_info in split_infos
+ ]
+
+ def _generate_shards(self, files):
+ yield from files
+
+ def _generate_tables(self, files):
+ # used to stream from cache
+ for file_idx, file in enumerate(files):
+ with open(file, "rb") as f:
+ try:
+ for batch_idx, record_batch in enumerate(pa.ipc.open_stream(f)):
+ pa_table = pa.Table.from_batches([record_batch])
+ # Uncomment for debugging (will print the Arrow table size and elements)
+ # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
+ # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
+ yield Key(file_idx, batch_idx), pa_table
+ except ValueError as e:
+ logger.error(f"Failed to read file '{file}' with error {type(e)}: {e}")
+ raise
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/conll/__init__.py b/venv/Lib/site-packages/datasets/packaged_modules/conll/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/conll/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/conll/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b840b08476e3fd6a18a9ca0e82634e0928cb6b91
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/conll/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/conll/__pycache__/conll.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/conll/__pycache__/conll.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..aa33e75675ea121289ac9f2317552eb4e12f0a0a
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/conll/__pycache__/conll.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/conll/conll.py b/venv/Lib/site-packages/datasets/packaged_modules/conll/conll.py
new file mode 100644
index 0000000000000000000000000000000000000000..d86d4b43e329cb9e16fd414d4518c27d3d585436
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/conll/conll.py
@@ -0,0 +1,155 @@
+"""CoNLL format dataset builder.
+
+Reads CoNLL-style files where each line carries one token plus its tag columns,
+columns are whitespace-separated, and empty lines mark sentence boundaries.
+Each example is one sentence, with each configured column produced as a list
+aligned with the tokens list.
+
+Supports CoNLL-2000 (chunking), CoNLL-2003 (NER), CoNLL-U (Universal
+Dependencies), and any custom column schema by overriding ``column_names``.
+"""
+
+from dataclasses import dataclass, field
+from typing import Optional
+
+import pyarrow as pa
+
+import datasets
+from datasets.builder import Key
+from datasets.features.features import require_storage_cast
+from datasets.table import table_cast
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+DEFAULT_COLUMN_NAMES = ["tokens"]
+
+
+@dataclass
+class ConllConfig(datasets.BuilderConfig):
+ """BuilderConfig for CoNLL-style files.
+
+ Args:
+ features: (`Features`, *optional*):
+ Cast the data to `features`.
+ column_names: (`list[str]`, defaults to `["tokens"]`):
+ Names for each whitespace-separated column. The first name is the
+ token column; subsequent names cover the tag columns. Common
+ schemas:
+
+ - CoNLL-2003 NER: `["tokens", "pos_tags", "chunk_tags", "ner_tags"]`
+ - CoNLL-2000 chunking: `["tokens", "pos_tags", "chunk_tags"]`
+ - CoNLL-U: `["id", "form", "lemma", "upos", "xpos", "feats",
+ "head", "deprel", "deps", "misc"]`
+ delimiter: (`str`, *optional*):
+ Column delimiter inside each line. If `None` (default), any
+ whitespace is used, matching the standard CoNLL convention.
+ encoding: (`str`, defaults to `"utf-8"`):
+ Encoding to decode the file.
+ encoding_errors: (`str`, *optional*):
+ Argument to define what to do in case of encoding error. Same as
+ the `errors` argument in `open()`.
+ skip_docstart: (`bool`, defaults to `True`):
+ Skip CoNLL-2003 `-DOCSTART-` document-boundary marker lines.
+ comment_prefix: (`str`, *optional*):
+ If set, lines starting with this prefix are skipped. CoNLL-U
+ comments start with `#`.
+ """
+
+ features: Optional[datasets.Features] = None
+ column_names: list[str] = field(default_factory=lambda: list(DEFAULT_COLUMN_NAMES))
+ delimiter: Optional[str] = None
+ encoding: str = "utf-8"
+ encoding_errors: Optional[str] = None
+ skip_docstart: bool = True
+ comment_prefix: Optional[str] = None
+
+
+class Conll(datasets.ArrowBasedBuilder):
+ BUILDER_CONFIG_CLASS = ConllConfig
+
+ def _info(self):
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager):
+ """The `data_files` kwarg in load_dataset() can be a str, List[str],
+ Dict[str,str], or Dict[str,List[str]].
+
+ If str or List[str], then the dataset returns only the 'train' split.
+ If dict, then keys should be from the `datasets.Split` enum.
+ """
+ if not self.config.data_files:
+ raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+ dl_manager.download_config.extract_on_the_fly = True
+ base_data_files = dl_manager.download(self.config.data_files)
+ extracted_data_files = dl_manager.extract(base_data_files)
+ splits = []
+ for split_name, files in extracted_data_files.items():
+ files_iterables = [dl_manager.iter_files(file) for file in files]
+ splits.append(
+ datasets.SplitGenerator(
+ name=split_name,
+ gen_kwargs={
+ "files_iterables": files_iterables,
+ "base_files": base_data_files[split_name],
+ },
+ )
+ )
+ return splits
+
+ def _cast_table(self, pa_table: pa.Table) -> pa.Table:
+ if self.config.features is not None:
+ schema = self.config.features.arrow_schema
+ if all(not require_storage_cast(feature) for feature in self.config.features.values()):
+ pa_table = pa_table.cast(schema)
+ else:
+ pa_table = table_cast(pa_table, schema)
+ return pa_table
+
+ def _generate_shards(self, base_files, files_iterables):
+ yield from base_files
+
+ def _generate_tables(self, base_files, files_iterables):
+ column_names = list(self.config.column_names)
+ if not column_names:
+ raise ValueError("ConllConfig.column_names must be a non-empty list")
+ num_cols = len(column_names)
+ delimiter = self.config.delimiter
+ skip_docstart = self.config.skip_docstart
+ comment_prefix = self.config.comment_prefix
+
+ for shard_idx, files_iterable in enumerate(files_iterables):
+ for file in files_iterable:
+ sentences: list[list[list[str]]] = []
+ current: list[list[str]] = [[] for _ in range(num_cols)]
+ with open(file, encoding=self.config.encoding, errors=self.config.encoding_errors) as f:
+ for raw_line in f:
+ # Strip the trailing newline only — preserve embedded whitespace
+ line = raw_line.rstrip("\r\n")
+ if not line.strip():
+ # Sentence boundary
+ if current[0]:
+ sentences.append(current)
+ current = [[] for _ in range(num_cols)]
+ continue
+ if skip_docstart and line.startswith("-DOCSTART-"):
+ continue
+ if comment_prefix is not None and line.startswith(comment_prefix):
+ continue
+ parts = line.split(delimiter) if delimiter is not None else line.split()
+ # Pad or truncate to num_cols so column alignment is preserved
+ if len(parts) < num_cols:
+ parts = parts + [""] * (num_cols - len(parts))
+ elif len(parts) > num_cols:
+ parts = parts[:num_cols]
+ for i, val in enumerate(parts):
+ current[i].append(val)
+ # Tail sentence with no trailing blank line
+ if current[0]:
+ sentences.append(current)
+
+ if sentences:
+ arrays = [pa.array([sentence[col_idx] for sentence in sentences]) for col_idx in range(num_cols)]
+ pa_table = pa.Table.from_arrays(arrays, names=column_names)
+ yield Key(shard_idx, 0), self._cast_table(pa_table)
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/csv/__init__.py b/venv/Lib/site-packages/datasets/packaged_modules/csv/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/csv/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/csv/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0793af2c894e2b5b919173367a0f4eddefd9e460
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/csv/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/csv/__pycache__/csv.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/csv/__pycache__/csv.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9fe12da76f82c0f0efc4e0d46d0998345caa6a04
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/csv/__pycache__/csv.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/csv/csv.py b/venv/Lib/site-packages/datasets/packaged_modules/csv/csv.py
new file mode 100644
index 0000000000000000000000000000000000000000..249df36e2950025c0a05c298752de1c1dd5df7b1
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/csv/csv.py
@@ -0,0 +1,206 @@
+from dataclasses import dataclass
+from typing import Any, Callable, Optional, Union
+
+import pandas as pd
+import pyarrow as pa
+
+import datasets
+import datasets.config
+from datasets.builder import Key
+from datasets.features.features import require_storage_cast
+from datasets.table import table_cast
+from datasets.utils.py_utils import Literal
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+_PANDAS_READ_CSV_NO_DEFAULT_PARAMETERS = ["names", "prefix"]
+_PANDAS_READ_CSV_DEPRECATED_PARAMETERS = ["warn_bad_lines", "error_bad_lines", "mangle_dupe_cols"]
+_PANDAS_READ_CSV_NEW_1_3_0_PARAMETERS = ["encoding_errors", "on_bad_lines"]
+_PANDAS_READ_CSV_NEW_2_0_0_PARAMETERS = ["date_format"]
+_PANDAS_READ_CSV_DEPRECATED_2_2_0_PARAMETERS = ["verbose"]
+
+
+@dataclass
+class CsvConfig(datasets.BuilderConfig):
+ """BuilderConfig for CSV."""
+
+ sep: str = ","
+ delimiter: Optional[str] = None
+ header: Optional[Union[int, list[int], str]] = "infer"
+ names: Optional[list[str]] = None
+ column_names: Optional[list[str]] = None
+ index_col: Optional[Union[int, str, list[int], list[str]]] = None
+ usecols: Optional[Union[list[int], list[str]]] = None
+ prefix: Optional[str] = None
+ mangle_dupe_cols: bool = True
+ engine: Optional[Literal["c", "python", "pyarrow"]] = None
+ converters: dict[Union[int, str], Callable[[Any], Any]] = None
+ true_values: Optional[list] = None
+ false_values: Optional[list] = None
+ skipinitialspace: bool = False
+ skiprows: Optional[Union[int, list[int]]] = None
+ nrows: Optional[int] = None
+ na_values: Optional[Union[str, list[str]]] = None
+ keep_default_na: bool = True
+ na_filter: bool = True
+ verbose: bool = False
+ skip_blank_lines: bool = True
+ thousands: Optional[str] = None
+ decimal: str = "."
+ lineterminator: Optional[str] = None
+ quotechar: str = '"'
+ quoting: int = 0
+ escapechar: Optional[str] = None
+ comment: Optional[str] = None
+ encoding: Optional[str] = None
+ dialect: Optional[str] = None
+ error_bad_lines: bool = True
+ warn_bad_lines: bool = True
+ skipfooter: int = 0
+ doublequote: bool = True
+ memory_map: bool = False
+ float_precision: Optional[str] = None
+ chunksize: int = 10_000
+ features: Optional[datasets.Features] = None
+ encoding_errors: Optional[str] = "strict"
+ on_bad_lines: Literal["error", "warn", "skip"] = "error"
+ date_format: Optional[str] = None
+
+ def __post_init__(self):
+ super().__post_init__()
+ if self.delimiter is not None:
+ self.sep = self.delimiter
+ if self.column_names is not None:
+ self.names = self.column_names
+
+ @property
+ def pd_read_csv_kwargs(self):
+ pd_read_csv_kwargs = {
+ "sep": self.sep,
+ "header": self.header,
+ "names": self.names,
+ "index_col": self.index_col,
+ "usecols": self.usecols,
+ "prefix": self.prefix,
+ "mangle_dupe_cols": self.mangle_dupe_cols,
+ "engine": self.engine,
+ "converters": self.converters,
+ "true_values": self.true_values,
+ "false_values": self.false_values,
+ "skipinitialspace": self.skipinitialspace,
+ "skiprows": self.skiprows,
+ "nrows": self.nrows,
+ "na_values": self.na_values,
+ "keep_default_na": self.keep_default_na,
+ "na_filter": self.na_filter,
+ "verbose": self.verbose,
+ "skip_blank_lines": self.skip_blank_lines,
+ "thousands": self.thousands,
+ "decimal": self.decimal,
+ "lineterminator": self.lineterminator,
+ "quotechar": self.quotechar,
+ "quoting": self.quoting,
+ "escapechar": self.escapechar,
+ "comment": self.comment,
+ "encoding": self.encoding,
+ "dialect": self.dialect,
+ "error_bad_lines": self.error_bad_lines,
+ "warn_bad_lines": self.warn_bad_lines,
+ "skipfooter": self.skipfooter,
+ "doublequote": self.doublequote,
+ "memory_map": self.memory_map,
+ "float_precision": self.float_precision,
+ "chunksize": self.chunksize,
+ "encoding_errors": self.encoding_errors,
+ "on_bad_lines": self.on_bad_lines,
+ "date_format": self.date_format,
+ }
+
+ # some kwargs must not be passed if they don't have a default value
+ # some others are deprecated and we can also not pass them if they are the default value
+ for pd_read_csv_parameter in _PANDAS_READ_CSV_NO_DEFAULT_PARAMETERS + _PANDAS_READ_CSV_DEPRECATED_PARAMETERS:
+ if pd_read_csv_kwargs[pd_read_csv_parameter] == getattr(CsvConfig(), pd_read_csv_parameter):
+ del pd_read_csv_kwargs[pd_read_csv_parameter]
+
+ # Remove 1.3 new arguments
+ if not (datasets.config.PANDAS_VERSION.major >= 1 and datasets.config.PANDAS_VERSION.minor >= 3):
+ for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_1_3_0_PARAMETERS:
+ del pd_read_csv_kwargs[pd_read_csv_parameter]
+
+ # Remove 2.0 new arguments
+ if not (datasets.config.PANDAS_VERSION.major >= 2):
+ for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_2_0_0_PARAMETERS:
+ del pd_read_csv_kwargs[pd_read_csv_parameter]
+
+ # Remove 2.2 deprecated arguments
+ if datasets.config.PANDAS_VERSION.release >= (2, 2):
+ for pd_read_csv_parameter in _PANDAS_READ_CSV_DEPRECATED_2_2_0_PARAMETERS:
+ if pd_read_csv_kwargs[pd_read_csv_parameter] == getattr(CsvConfig(), pd_read_csv_parameter):
+ del pd_read_csv_kwargs[pd_read_csv_parameter]
+
+ return pd_read_csv_kwargs
+
+
+class Csv(datasets.ArrowBasedBuilder):
+ BUILDER_CONFIG_CLASS = CsvConfig
+
+ def _info(self):
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager):
+ """We handle string, list and dicts in datafiles"""
+ if not self.config.data_files:
+ raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+ dl_manager.download_config.extract_on_the_fly = True
+ base_data_files = dl_manager.download(self.config.data_files)
+ extracted_data_files = dl_manager.extract(base_data_files)
+ splits = []
+ for split_name, extracted_files in extracted_data_files.items():
+ files_iterables = [dl_manager.iter_files(extracted_file) for extracted_file in extracted_files]
+ splits.append(
+ datasets.SplitGenerator(
+ name=split_name,
+ gen_kwargs={"files_iterables": files_iterables, "base_files": base_data_files[split_name]},
+ )
+ )
+ return splits
+
+ def _cast_table(self, pa_table: pa.Table) -> pa.Table:
+ if self.config.features is not None:
+ schema = self.config.features.arrow_schema
+ if all(not require_storage_cast(feature) for feature in self.config.features.values()):
+ # cheaper cast
+ pa_table = pa.Table.from_arrays([pa_table[field.name] for field in schema], schema=schema)
+ else:
+ # more expensive cast; allows str <-> int/float or str to Audio for example
+ pa_table = table_cast(pa_table, schema)
+ return pa_table
+
+ def _generate_shards(self, base_files, files_iterables):
+ yield from base_files
+
+ def _generate_tables(self, base_files, files_iterables):
+ schema = self.config.features.arrow_schema if self.config.features else None
+ # dtype allows reading an int column as str
+ dtype = (
+ {
+ name: dtype.to_pandas_dtype() if not require_storage_cast(feature) else object
+ for name, dtype, feature in zip(schema.names, schema.types, self.config.features.values())
+ }
+ if schema is not None
+ else None
+ )
+ for shard_idx, files_iterable in enumerate(files_iterables):
+ for file in files_iterable:
+ csv_file_reader = pd.read_csv(file, iterator=True, dtype=dtype, **self.config.pd_read_csv_kwargs)
+ try:
+ for batch_idx, df in enumerate(csv_file_reader):
+ pa_table = pa.Table.from_pandas(df)
+ # Uncomment for debugging (will print the Arrow table size and elements)
+ # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
+ # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
+ yield Key(shard_idx, batch_idx), self._cast_table(pa_table)
+ except ValueError as e:
+ logger.error(f"Failed to read file '{file}' with error {type(e)}: {e}")
+ raise
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/eval/__init__.py b/venv/Lib/site-packages/datasets/packaged_modules/eval/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/eval/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/eval/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..fd30e769a29f673668c6661b56f5c00b7f311f6b
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/eval/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/eval/__pycache__/eval.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/eval/__pycache__/eval.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ea5351ffeae5496edbb87dccdee63fc510d9ffde
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/eval/__pycache__/eval.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/eval/eval.py b/venv/Lib/site-packages/datasets/packaged_modules/eval/eval.py
new file mode 100644
index 0000000000000000000000000000000000000000..f2198dd99bdaa41bdf41e0dc4872ee13c36ede0d
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/eval/eval.py
@@ -0,0 +1,77 @@
+import json
+import os
+from itertools import islice
+from typing import Iterable
+
+import pyarrow as pa
+
+import datasets
+from datasets.builder import Key
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+class Eval(datasets.GeneratorBasedBuilder):
+ NUM_EXAMPLES_FOR_FEATURES_INFERENCE = 5
+
+ def _info(self):
+ return datasets.DatasetInfo()
+
+ def _split_generators(self, dl_manager):
+ """We handle string, list and dicts in datafiles"""
+ if not self.config.data_files:
+ raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+ dl_manager.download_config.extract_on_the_fly = True
+ base_data_files = dl_manager.download(self.config.data_files)
+ extracted_data_files = dl_manager.extract(base_data_files)
+ splits = []
+ for split_name, logs in extracted_data_files.items():
+ logs_files_iterables = [dl_manager.iter_files(log) for log in logs]
+ splits.append(
+ datasets.SplitGenerator(
+ name=split_name,
+ gen_kwargs={
+ "logs_files_iterables": logs_files_iterables,
+ "base_files": base_data_files[split_name],
+ },
+ )
+ )
+ if not self.info.features:
+ first_examples = list(
+ islice(
+ self._iter_samples_from_log_files(logs_files_iterables[0]),
+ self.NUM_EXAMPLES_FOR_FEATURES_INFERENCE,
+ )
+ )
+ pa_tables = [pa.Table.from_pylist([example]) for example in first_examples]
+ inferred_arrow_schema = pa.concat_tables(pa_tables, promote_options="default").schema
+ self.info.features = datasets.Features.from_arrow_schema(inferred_arrow_schema)
+
+ return splits
+
+ def _sort_samples_key(self, sample_path: str):
+ # looks like "{sample_idx}_epoch_{epoch_idx}""
+ (sample_idx_str, epoch_idx_str) = os.path.splitext(os.path.basename(sample_path))[0].split("_epoch_")
+ return (int(epoch_idx_str), int(sample_idx_str))
+
+ def _iter_samples_from_log_files(self, log_files: Iterable[str]):
+ sample_files = [log_file for log_file in log_files if os.path.basename(os.path.dirname(log_file)) == "samples"]
+ sample_files.sort(key=self._sort_samples_key)
+ for sample_file in sample_files:
+ with open(sample_file) as f:
+ sample = json.load(f)
+ for field in sample:
+ if isinstance(sample[field], dict):
+ sample[field] = json.dumps(sample[field])
+ if isinstance(sample[field], list):
+ sample[field] = [json.dumps(x) for x in sample[field]]
+ yield sample
+
+ def _generate_shards(self, base_files, logs_files_iterables):
+ yield from base_files
+
+ def _generate_examples(self, base_files, logs_files_iterables):
+ for file_idx, log_files in enumerate(logs_files_iterables):
+ for sample_idx, sample in enumerate(self._iter_samples_from_log_files(log_files)):
+ yield Key(file_idx, sample_idx), sample
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/folder_based_builder/__init__.py b/venv/Lib/site-packages/datasets/packaged_modules/folder_based_builder/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ef21bc9362d146b574f5eda4d56988f2b21189bb
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/folder_based_builder.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/folder_based_builder.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1419bde6bcb237278e46d5a79d43803e4060afbb
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/folder_based_builder.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/folder_based_builder/folder_based_builder.py b/venv/Lib/site-packages/datasets/packaged_modules/folder_based_builder/folder_based_builder.py
new file mode 100644
index 0000000000000000000000000000000000000000..82586c648cbf6039625ac1de7dfc50b2106ca7a7
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/folder_based_builder/folder_based_builder.py
@@ -0,0 +1,444 @@
+import collections
+import io
+import itertools
+import os
+from dataclasses import dataclass
+from typing import Any, Callable, Iterator, Optional, Union
+
+import pandas as pd
+import pyarrow as pa
+import pyarrow.dataset as ds
+import pyarrow.json as paj
+import pyarrow.parquet as pq
+
+import datasets
+from datasets import config
+from datasets.builder import Key
+from datasets.features.features import FeatureType, _visit, _visit_with_path, _VisitPath, require_storage_cast
+from datasets.utils.file_utils import readline
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+def count_path_segments(path):
+ return path.replace("\\", "/").count("/")
+
+
+@dataclass
+class FolderBasedBuilderConfig(datasets.BuilderConfig):
+ """BuilderConfig for AutoFolder."""
+
+ features: Optional[datasets.Features] = None
+ drop_labels: bool = None
+ drop_metadata: bool = None
+ metadata_filenames: list[str] = None
+ filters: Optional[Union[ds.Expression, list[tuple], list[list[tuple]]]] = None
+
+ def __post_init__(self):
+ super().__post_init__()
+
+
+class FolderBasedBuilder(datasets.GeneratorBasedBuilder):
+ """
+ Base class for generic data loaders for vision and image data.
+
+
+ Abstract class attributes to be overridden by a child class:
+ BASE_FEATURE: feature object to decode data (i.e. datasets.Image, datasets.Audio, ...)
+ BASE_COLUMN_NAME: string key name of a base feature (i.e. "image", "audio", ...)
+ BUILDER_CONFIG_CLASS: builder config inherited from `folder_based_builder.FolderBasedBuilderConfig`
+ EXTENSIONS: list of allowed extensions (only files with these extensions and METADATA_FILENAME files
+ will be included in a dataset)
+ """
+
+ BASE_FEATURE: type[FeatureType]
+ BASE_COLUMN_NAME: str
+ BUILDER_CONFIG_CLASS: FolderBasedBuilderConfig
+ EXTENSIONS: list[str]
+
+ METADATA_FILENAMES: list[str] = ["metadata.csv", "metadata.jsonl", "metadata.parquet"]
+
+ def _info(self):
+ if not self.config.data_dir and not self.config.data_files:
+ raise ValueError(
+ "Folder-based datasets require either `data_dir` or `data_files` to be specified. "
+ "Neither was provided."
+ )
+
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager):
+ if not self.config.data_files:
+ raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+ dl_manager.download_config.extract_on_the_fly = True
+ # Do an early pass if:
+ # * `drop_labels` is None (default) or False, to infer the class labels
+ # * `drop_metadata` is None (default) or False, to find the metadata files
+ do_analyze = not self.config.drop_labels or not self.config.drop_metadata
+ labels, path_depths = set(), set()
+ all_metadata_files = collections.defaultdict(set)
+ metadata_filenames = self.config.metadata_filenames or self.METADATA_FILENAMES
+
+ def analyze(files_or_archives, downloaded_files_or_dirs, split):
+ if len(downloaded_files_or_dirs) == 0:
+ return
+ # The files are separated from the archives at this point, so check the first sample
+ # to see if it's a file or a directory and iterate accordingly
+ if os.path.isfile(downloaded_files_or_dirs[0]):
+ original_files, downloaded_files = files_or_archives, downloaded_files_or_dirs
+ for original_file, downloaded_file in zip(original_files, downloaded_files):
+ original_file, downloaded_file = str(original_file), str(downloaded_file)
+ _, original_file_ext = os.path.splitext(original_file)
+ if original_file_ext.lower() in self.EXTENSIONS:
+ if not self.config.drop_labels:
+ labels.add(os.path.basename(os.path.dirname(original_file)))
+ path_depths.add(count_path_segments(original_file))
+ elif os.path.basename(original_file) in metadata_filenames:
+ all_metadata_files[split].add((original_file, None, downloaded_file))
+ else:
+ original_file_name = os.path.basename(original_file)
+ logger.debug(
+ f"The file '{original_file_name}' was ignored: it is not a {self.BASE_COLUMN_NAME}, and is not {metadata_filenames} either."
+ )
+ else:
+ archives, downloaded_dirs = files_or_archives, downloaded_files_or_dirs
+ for archive, downloaded_dir in zip(archives, downloaded_dirs):
+ archive, downloaded_dir = str(archive), str(downloaded_dir)
+ for downloaded_dir_file in dl_manager.iter_files(downloaded_dir):
+ _, downloaded_dir_file_ext = os.path.splitext(downloaded_dir_file)
+ if downloaded_dir_file_ext in self.EXTENSIONS:
+ if not self.config.drop_labels:
+ labels.add(os.path.basename(os.path.dirname(downloaded_dir_file)))
+ path_depths.add(count_path_segments(downloaded_dir_file))
+ elif os.path.basename(downloaded_dir_file) in metadata_filenames:
+ all_metadata_files[split].add((None, downloaded_dir, downloaded_dir_file))
+ else:
+ archive_file_name = os.path.basename(archive)
+ original_file_name = os.path.basename(downloaded_dir_file)
+ logger.debug(
+ f"The file '{original_file_name}' from the archive '{archive_file_name}' was ignored: it is not a {self.BASE_COLUMN_NAME}, and is not {metadata_filenames} either."
+ )
+
+ data_files = self.config.data_files
+ splits = []
+ for split_name, files in data_files.items():
+ files, metadata_files, archives = self._split_files_and_metadata_and_archives(files)
+ downloaded_files = dl_manager.download(files)
+ downloaded_metadata_files = dl_manager.download(metadata_files)
+ downloaded_dirs = dl_manager.download_and_extract(archives)
+ if do_analyze: # drop_metadata is None or False, drop_labels is None or False
+ logger.info(f"Searching for labels and/or metadata files in {split_name} data files...")
+ analyze(files, downloaded_files, split_name)
+ analyze(metadata_files, downloaded_metadata_files, split_name)
+ analyze(archives, downloaded_dirs, split_name)
+
+ if all_metadata_files:
+ # add metadata if `all_metadata_files` are found and `drop_metadata` is None (default) or False
+ add_metadata = not self.config.drop_metadata
+ # if `all_metadata_files` are found, don't add labels
+ add_labels = False
+ else:
+ # if `all_metadata_files` are not found, don't add metadata
+ add_metadata = False
+ # if `all_metadata_files` are not found and `drop_labels` is None (default) -
+ # add labels if files are on the same level in directory hierarchy and there is more than one label
+ add_labels = (
+ (len(labels) > 1 and len(path_depths) == 1)
+ if self.config.drop_labels is None
+ else not self.config.drop_labels
+ )
+
+ if add_labels:
+ logger.info("Adding the labels inferred from data directories to the dataset's features...")
+ if add_metadata:
+ logger.info("Adding metadata to the dataset...")
+ else:
+ add_labels, add_metadata, all_metadata_files = False, False, {}
+
+ # files info (original_file, None, downloaded_file)
+ files = tuple(zip(files, [None] * len(files), downloaded_files))
+ # archives info (original_archive_file, downloaded_dir, downloaded_files)
+ files += tuple(
+ (archive, downloaded_dir, dl_manager.iter_files(downloaded_dir))
+ for archive, downloaded_dir in zip(archives, downloaded_dirs)
+ )
+ splits.append(
+ datasets.SplitGenerator(
+ name=split_name,
+ gen_kwargs={
+ "files": files,
+ "metadata_files": all_metadata_files.get(split_name, []),
+ "add_labels": add_labels,
+ "add_metadata": add_metadata,
+ },
+ )
+ )
+
+ if add_metadata:
+ # Verify that:
+ # * all metadata files have the same set of features in each split
+ # * the `file_name` key is one of the metadata keys and is of type string
+ features_per_metadata_file: list[tuple[str, datasets.Features]] = []
+
+ # Check that all metadata files share the same format
+ metadata_ext = {
+ os.path.splitext(original_metadata_file or downloaded_metadata_file)[-1]
+ for original_metadata_file, _, downloaded_metadata_file in itertools.chain.from_iterable(
+ all_metadata_files.values()
+ )
+ }
+ if len(metadata_ext) > 1:
+ raise ValueError(f"Found metadata files with different extensions: {list(metadata_ext)}")
+ metadata_ext = metadata_ext.pop()
+
+ for split_metadata_files in all_metadata_files.values():
+ pa_metadata_table = None
+ for _, _, downloaded_metadata_file in split_metadata_files:
+ for pa_metadata_table in self._read_metadata(downloaded_metadata_file, metadata_ext=metadata_ext):
+ break # just fetch the first rows
+ if pa_metadata_table is not None:
+ features_per_metadata_file.append(
+ (downloaded_metadata_file, datasets.Features.from_arrow_schema(pa_metadata_table.schema))
+ )
+ break # no need to fetch all the files
+ for downloaded_metadata_file, metadata_features in features_per_metadata_file:
+ if metadata_features != features_per_metadata_file[0][1]:
+ raise ValueError(
+ f"Metadata files {downloaded_metadata_file} and {features_per_metadata_file[0][0]} have different features: {features_per_metadata_file[0]} != {metadata_features}"
+ )
+ metadata_features = features_per_metadata_file[0][1]
+ feature_not_found = True
+
+ def _set_feature(feature):
+ nonlocal feature_not_found
+ if isinstance(feature, dict):
+ out = type(feature)()
+ for key in feature:
+ if (key == "file_name" or key.endswith("_file_name")) and (
+ feature[key] == datasets.Value("string") or feature[key] == datasets.Value("large_string")
+ ):
+ key = key[: -len("_file_name")] or self.BASE_COLUMN_NAME
+ out[key] = self.BASE_FEATURE()
+ feature_not_found = False
+ elif (key == "file_names" or key.endswith("_file_names")) and (
+ feature[key]
+ in [datasets.List(datasets.Value("string")), datasets.List(datasets.Value("large_string"))]
+ ):
+ key = key[: -len("_file_names")] or (self.BASE_COLUMN_NAME + "s")
+ out[key] = datasets.List(self.BASE_FEATURE())
+ feature_not_found = False
+ elif (key == "file_names" or key.endswith("_file_names")) and (
+ feature[key] == [datasets.Value("string")]
+ or feature[key] == [datasets.Value("large_string")]
+ ):
+ key = key[: -len("_file_names")] or (self.BASE_COLUMN_NAME + "s")
+ out[key] = [self.BASE_FEATURE()]
+ feature_not_found = False
+ else:
+ out[key] = feature[key]
+ return out
+ return feature
+
+ metadata_features = _visit(metadata_features, _set_feature)
+
+ if feature_not_found:
+ raise ValueError(
+ "`file_name`, `*_file_name`, `file_names` or `*_file_names` must be present as dictionary key in metadata files"
+ )
+ else:
+ metadata_features = None
+
+ # Normally, we would do this in _info, but we need to know the labels and/or metadata
+ # before building the features
+ if self.config.features is None:
+ if add_metadata:
+ self.info.features = metadata_features
+ elif add_labels:
+ self.info.features = datasets.Features(
+ {
+ self.BASE_COLUMN_NAME: self.BASE_FEATURE(),
+ "label": datasets.ClassLabel(names=sorted(labels)),
+ }
+ )
+ else:
+ self.info.features = datasets.Features({self.BASE_COLUMN_NAME: self.BASE_FEATURE()})
+
+ return splits
+
+ def _split_files_and_metadata_and_archives(self, data_files):
+ files, metadata_files, archives = [], [], []
+ metadata_filenames = self.config.metadata_filenames or self.METADATA_FILENAMES
+ for data_file in data_files:
+ data_file_root, data_file_ext = os.path.splitext(data_file)
+ _, second_data_file_ext = os.path.splitext(data_file_root)
+ if data_file_ext.lower() in self.EXTENSIONS or second_data_file_ext.lower() in self.EXTENSIONS:
+ files.append(data_file)
+ elif os.path.basename(data_file) in metadata_filenames:
+ metadata_files.append(data_file)
+ elif data_file_ext.lower() == ".zip":
+ archives.append(data_file)
+ return files, metadata_files, archives
+
+ def _read_metadata(self, metadata_file: str, metadata_ext: str = "") -> Iterator[pa.Table]:
+ """using the same logic as the Csv, Json and Parquet dataset builders to stream the data"""
+ if self.config.filters is not None:
+ filter_expr = (
+ pq.filters_to_expression(self.config.filters)
+ if isinstance(self.config.filters, list)
+ else self.config.filters
+ )
+ else:
+ filter_expr = None
+ if metadata_ext == ".csv":
+ chunksize = 10_000 # 10k lines
+ schema = self.config.features.arrow_schema if self.config.features else None
+ # dtype allows reading an int column as str
+ dtype = (
+ {
+ name: dtype.to_pandas_dtype() if not require_storage_cast(feature) else object
+ for name, dtype, feature in zip(schema.names, schema.types, self.config.features.values())
+ }
+ if schema is not None
+ else None
+ )
+ csv_file_reader = pd.read_csv(metadata_file, iterator=True, dtype=dtype, chunksize=chunksize)
+ for df in csv_file_reader:
+ pa_table = pa.Table.from_pandas(df)
+ if self.config.filters is not None:
+ pa_table = pa_table.filter(filter_expr)
+ if len(pa_table) > 0:
+ yield pa_table
+ elif metadata_ext == ".jsonl":
+ with open(metadata_file, "rb") as f:
+ chunksize: int = 10 << 20 # 10MB
+ # Use block_size equal to the chunk size divided by 32 to leverage multithreading
+ # Set a default minimum value of 16kB if the chunk size is really small
+ block_size = max(chunksize // 32, 16 << 10)
+ while True:
+ batch = f.read(chunksize)
+ if not batch:
+ break
+ # Finish current line
+ try:
+ batch += f.readline()
+ except (AttributeError, io.UnsupportedOperation):
+ batch += readline(f)
+ while True:
+ try:
+ pa_table = paj.read_json(
+ io.BytesIO(batch), read_options=paj.ReadOptions(block_size=block_size)
+ )
+ break
+ except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
+ if (
+ isinstance(e, pa.ArrowInvalid)
+ and "straddling" not in str(e)
+ or block_size > len(batch)
+ ):
+ raise
+ else:
+ # Increase the block size in case it was too small.
+ # The block size will be reset for the next file.
+ logger.debug(
+ f"Batch of {len(batch)} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."
+ )
+ block_size *= 2
+ if self.config.filters is not None:
+ pa_table = pa_table.filter(filter_expr)
+ if len(pa_table) > 0:
+ yield pa_table
+ else:
+ with open(metadata_file, "rb") as f:
+ parquet_fragment = ds.ParquetFileFormat().make_fragment(f)
+ if parquet_fragment.row_groups:
+ batch_size = parquet_fragment.row_groups[0].num_rows
+ else:
+ batch_size = config.DEFAULT_MAX_BATCH_SIZE
+ for record_batch in parquet_fragment.to_batches(
+ batch_size=batch_size,
+ filter=filter_expr,
+ batch_readahead=0,
+ fragment_readahead=0,
+ ):
+ yield pa.Table.from_batches([record_batch])
+
+ def _generate_shards(self, files, metadata_files, add_metadata, add_labels):
+ if add_metadata:
+ for _, _, downloaded_metadata_file in metadata_files:
+ yield downloaded_metadata_file
+ else:
+ for _, downloaded_dir, downloaded_file in files:
+ yield downloaded_dir or downloaded_file
+
+ def _generate_examples(self, files, metadata_files, add_metadata, add_labels):
+ if add_metadata:
+ feature_paths = []
+
+ def find_feature_path(feature, feature_path):
+ nonlocal feature_paths
+ if feature_path and isinstance(feature, self.BASE_FEATURE):
+ feature_paths.append(feature_path)
+
+ _visit_with_path(self.info.features, find_feature_path)
+
+ for shard_idx, metadata_file_info in enumerate(metadata_files):
+ if len(metadata_file_info) == 2:
+ original_metadata_file, downloaded_metadata_file = metadata_file_info
+ else:
+ original_metadata_file, downloaded_metadata_dir, downloaded_metadata_file = metadata_file_info
+ metadata_ext = os.path.splitext(original_metadata_file or downloaded_metadata_file)[-1]
+ downloaded_metadata_dir = os.path.dirname(downloaded_metadata_file)
+
+ def set_feature(item, feature_path: _VisitPath):
+ if len(feature_path) == 2 and isinstance(feature_path[0], str) and feature_path[1] == 0:
+ item[feature_path[0]] = item.pop("file_names", None) or item.pop(
+ feature_path[0] + "_file_names", None
+ )
+ elif len(feature_path) == 1 and isinstance(feature_path[0], str):
+ item[feature_path[0]] = item.pop("file_name", None) or item.pop(
+ feature_path[0] + "_file_name", None
+ )
+ elif len(feature_path) == 0:
+ if item is not None:
+ file_relpath = os.path.normpath(item).replace("\\", "/")
+ item = os.path.join(downloaded_metadata_dir, file_relpath)
+ return item
+
+ for pa_metadata_table in self._read_metadata(downloaded_metadata_file, metadata_ext=metadata_ext):
+ for sample_idx, sample in enumerate(pa_metadata_table.to_pylist()):
+ for feature_path in feature_paths:
+ _nested_apply(sample, feature_path, set_feature)
+ yield Key(shard_idx, sample_idx), sample
+ else:
+ if self.config.filters is not None:
+ filter_expr = (
+ pq.filters_to_expression(self.config.filters)
+ if isinstance(self.config.filters, list)
+ else self.config.filters
+ )
+ for shard_idx, (original_file, _, downloaded_files) in enumerate(files):
+ if isinstance(downloaded_files, str):
+ downloaded_files = [downloaded_files]
+ for sample_idx, downloaded_file in enumerate(downloaded_files):
+ sample = {self.BASE_COLUMN_NAME: downloaded_file}
+ if add_labels:
+ sample["label"] = os.path.basename(os.path.dirname(original_file or downloaded_file))
+ if self.config.filters is not None:
+ pa_table = pa.Table.from_pylist([sample]).filter(filter_expr)
+ if len(pa_table) == 0:
+ continue
+ yield Key(shard_idx, sample_idx), sample
+
+
+def _nested_apply(item: Any, feature_path: _VisitPath, func: Callable[[Any, _VisitPath], Any]):
+ # see _visit_with_path() to see how feature paths are constructed
+ item = func(item, feature_path)
+ if feature_path:
+ key = feature_path[0]
+ if key == 0:
+ for i in range(len(item)):
+ item[i] = _nested_apply(item[i], feature_path[1:], func)
+ else:
+ item[key] = _nested_apply(item[key], feature_path[1:], func)
+ return item
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/generator/__init__.py b/venv/Lib/site-packages/datasets/packaged_modules/generator/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/generator/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/generator/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..842b0f2e5a4fb0ba18f1a14ed90d7af94bc09479
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/generator/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/generator/__pycache__/generator.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/generator/__pycache__/generator.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c7c229cf0e1f6f693cfd22dcc911ee3384cfa21f
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/generator/__pycache__/generator.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/generator/generator.py b/venv/Lib/site-packages/datasets/packaged_modules/generator/generator.py
new file mode 100644
index 0000000000000000000000000000000000000000..e20cada94efd6cf33bbe02b05438eced4e967f54
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/generator/generator.py
@@ -0,0 +1,38 @@
+from dataclasses import dataclass
+from typing import Callable, Optional
+
+import datasets
+from datasets.builder import Key
+from datasets.utils.sharding import _number_of_shards_in_gen_kwargs, _split_gen_kwargs
+
+
+@dataclass
+class GeneratorConfig(datasets.BuilderConfig):
+ generator: Optional[Callable] = None
+ gen_kwargs: Optional[dict] = None
+ features: Optional[datasets.Features] = None
+ split: datasets.NamedSplit = datasets.Split.TRAIN
+
+ def __post_init__(self):
+ super().__post_init__()
+ if self.generator is None:
+ raise ValueError("generator must be specified")
+
+ if self.gen_kwargs is None:
+ self.gen_kwargs = {}
+
+
+class Generator(datasets.GeneratorBasedBuilder):
+ BUILDER_CONFIG_CLASS = GeneratorConfig
+
+ def _info(self):
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager):
+ return [datasets.SplitGenerator(name=self.config.split, gen_kwargs=self.config.gen_kwargs)]
+
+ def _generate_examples(self, **gen_kwargs):
+ num_shards = _number_of_shards_in_gen_kwargs(gen_kwargs)
+ for shard_idx, shard_gen_kwargs in enumerate(_split_gen_kwargs(gen_kwargs, max_num_jobs=num_shards)):
+ for sample_idx, sample in enumerate(self.config.generator(**shard_gen_kwargs)):
+ yield Key(shard_idx, sample_idx), sample
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/hdf5/__init__.py b/venv/Lib/site-packages/datasets/packaged_modules/hdf5/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/hdf5/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/hdf5/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6ca50d560405a5b4f853824b413bd75d38d4cf9c
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/hdf5/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/hdf5/__pycache__/hdf5.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/hdf5/__pycache__/hdf5.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..23db6c15e496bd1293b42daaff70c029316fab35
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/hdf5/__pycache__/hdf5.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/hdf5/hdf5.py b/venv/Lib/site-packages/datasets/packaged_modules/hdf5/hdf5.py
new file mode 100644
index 0000000000000000000000000000000000000000..e06b2bbb8f0d5c51d9f5863e536dc851a4cbb9b8
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/hdf5/hdf5.py
@@ -0,0 +1,389 @@
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Optional
+
+import numpy as np
+import pyarrow as pa
+
+import datasets
+from datasets.builder import Key
+from datasets.features.features import (
+ Array2D,
+ Array3D,
+ Array4D,
+ Array5D,
+ Features,
+ LargeList,
+ List,
+ Value,
+ _ArrayXD,
+ _arrow_to_datasets_dtype,
+)
+from datasets.table import cast_table_to_features
+
+
+if TYPE_CHECKING:
+ import h5py
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+EXTENSIONS = [".h5", ".hdf5"]
+
+
+@dataclass
+class HDF5Config(datasets.BuilderConfig):
+ """BuilderConfig for HDF5."""
+
+ batch_size: Optional[int] = None
+ features: Optional[datasets.Features] = None
+
+
+class HDF5(datasets.ArrowBasedBuilder):
+ """ArrowBasedBuilder that converts HDF5 files to Arrow tables using the HF extension types."""
+
+ BUILDER_CONFIG_CLASS = HDF5Config
+
+ def _info(self):
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager):
+ import h5py
+
+ if not self.config.data_files:
+ raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+ data_files = dl_manager.download(self.config.data_files)
+ splits = []
+ for split_name, files in data_files.items():
+ # Infer features from first file
+ if self.info.features is None:
+ for first_file in files:
+ with open(first_file, "rb") as f:
+ with h5py.File(f, "r") as h5:
+ self.info.features = _recursive_infer_features(h5)
+ break
+ splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files}))
+ return splits
+
+ def _generate_shards(self, files):
+ yield from files
+
+ def _generate_tables(self, files):
+ import h5py
+
+ batch_size_cfg = self.config.batch_size
+ for file_idx, file in enumerate(files):
+ try:
+ with open(file, "rb") as f:
+ with h5py.File(f, "r") as h5:
+ # Infer features and lengths from first file
+ if self.info.features is None:
+ self.info.features = _recursive_infer_features(h5)
+ num_rows = _check_dataset_lengths(h5, self.info.features)
+ if num_rows is None:
+ logger.warning(f"File {file} contains no data, skipping...")
+ continue
+ effective_batch = batch_size_cfg or self._writer_batch_size or num_rows
+ for batch_idx, start in enumerate(range(0, num_rows, effective_batch)):
+ end = min(start + effective_batch, num_rows)
+ pa_table = _recursive_load_arrays(h5, self.info.features, start, end)
+ if pa_table is None:
+ logger.warning(f"File {file} contains no data, skipping...")
+ continue
+ yield Key(file_idx, batch_idx), cast_table_to_features(pa_table, self.info.features)
+ except ValueError as e:
+ logger.error(f"Failed to read file '{file}' with error {type(e)}: {e}")
+ raise
+
+
+# ┌───────────┐
+# │ Complex │
+# └───────────┘
+
+
+def _is_complex_dtype(dtype: np.dtype) -> bool:
+ if dtype.kind == "c":
+ return True
+ if dtype.subdtype is not None:
+ return _is_complex_dtype(dtype.subdtype[0])
+ return False
+
+
+def _create_complex_features(dset) -> Features:
+ if dset.dtype.subdtype is not None:
+ dtype, data_shape = dset.dtype.subdtype
+ else:
+ data_shape = dset.shape[1:]
+ dtype = dset.dtype
+
+ if dtype == np.complex64:
+ # two float32s
+ value_type = Value("float32")
+ elif dtype == np.complex128:
+ # two float64s
+ value_type = Value("float64")
+ else:
+ logger.warning(f"Found complex dtype {dtype} that is not supported. Converting to float64...")
+ value_type = Value("float64")
+
+ return Features(
+ {
+ "real": _create_sized_feature_impl(data_shape, value_type),
+ "imag": _create_sized_feature_impl(data_shape, value_type),
+ }
+ )
+
+
+def _convert_complex_to_nested(arr: np.ndarray) -> pa.StructArray:
+ data = {
+ "real": datasets.features.features.numpy_to_pyarrow_listarray(arr.real),
+ "imag": datasets.features.features.numpy_to_pyarrow_listarray(arr.imag),
+ }
+ return pa.StructArray.from_arrays([data["real"], data["imag"]], names=["real", "imag"])
+
+
+# ┌────────────┐
+# │ Compound │
+# └────────────┘
+
+
+def _is_compound_dtype(dtype: np.dtype) -> bool:
+ return dtype.kind == "V"
+
+
+@dataclass
+class _CompoundGroup:
+ dset: "h5py.Dataset"
+ data: np.ndarray = None
+
+ def items(self):
+ for field_name in self.dset.dtype.names:
+ field_dtype = self.dset.dtype[field_name]
+ yield field_name, _CompoundField(self.data, field_name, field_dtype)
+
+
+@dataclass
+class _CompoundField:
+ data: Optional[np.ndarray]
+ name: str
+ dtype: np.dtype
+ shape: tuple[int, ...] = field(init=False)
+
+ def __post_init__(self):
+ self.shape = (len(self.data) if self.data is not None else 0,) + self.dtype.shape
+
+ def __getitem__(self, key):
+ return self.data[key][self.name]
+
+
+def _create_compound_features(dset) -> Features:
+ mock_group = _CompoundGroup(dset)
+ return _recursive_infer_features(mock_group)
+
+
+def _convert_compound_to_nested(arr, dset) -> pa.StructArray:
+ mock_group = _CompoundGroup(dset, data=arr)
+ features = _create_compound_features(dset)
+ return _recursive_load_arrays(mock_group, features, 0, len(arr))
+
+
+# ┌───────────────────┐
+# │ Variable-Length │
+# └───────────────────┘
+
+
+def _is_vlen_dtype(dtype: np.dtype) -> bool:
+ if dtype.metadata and "vlen" in dtype.metadata:
+ return True
+ return False
+
+
+def _create_vlen_features(dset) -> Features:
+ vlen_dtype = dset.dtype.metadata["vlen"]
+ if vlen_dtype in (str, bytes):
+ return Value("string")
+ inner_feature = _np_to_pa_to_hf_value(vlen_dtype)
+ return List(inner_feature)
+
+
+def _convert_vlen_to_array(arr: np.ndarray) -> pa.Array:
+ return datasets.features.features.numpy_to_pyarrow_listarray(arr)
+
+
+# ┌───────────┐
+# │ Generic │
+# └───────────┘
+
+
+def _recursive_infer_features(h5_obj) -> Features:
+ features_dict = {}
+ for path, dset in h5_obj.items():
+ if _is_group(dset):
+ features = _recursive_infer_features(dset)
+ if features:
+ features_dict[path] = features
+ elif _is_dataset(dset):
+ features = _infer_feature(dset)
+ if features:
+ features_dict[path] = features
+
+ return Features(features_dict)
+
+
+def _infer_feature(dset):
+ if _is_complex_dtype(dset.dtype):
+ return _create_complex_features(dset)
+ elif _is_compound_dtype(dset.dtype) or dset.dtype.kind == "V":
+ return _create_compound_features(dset)
+ elif _is_vlen_dtype(dset.dtype):
+ return _create_vlen_features(dset)
+ return _create_sized_feature(dset)
+
+
+def _load_array(dset, path: str, start: int, end: int) -> pa.Array:
+ arr = dset[start:end]
+
+ if _is_vlen_dtype(dset.dtype):
+ return _convert_vlen_to_array(arr)
+ elif _is_complex_dtype(dset.dtype):
+ return _convert_complex_to_nested(arr)
+ elif _is_compound_dtype(dset.dtype):
+ return _convert_compound_to_nested(arr, dset)
+ elif dset.dtype.kind == "O":
+ raise ValueError(
+ f"Object dtype dataset '{path}' is not supported. "
+ f"For variable-length data, please use h5py.vlen_dtype() "
+ f"when creating the HDF5 file. "
+ f"See: https://docs.h5py.org/en/stable/special.html#variable-length-strings"
+ )
+ else:
+ # If any non-batch dimension is zero, emit an unsized pa.list_
+ # to avoid creating FixedSizeListArray with list_size=0.
+ if any(dim == 0 for dim in dset.shape[1:]):
+ inner_type = pa.from_numpy_dtype(dset.dtype)
+ return pa.array([[] for _ in arr], type=pa.list_(inner_type))
+ else:
+ return datasets.features.features.numpy_to_pyarrow_listarray(arr)
+
+
+def _recursive_load_arrays(h5_obj, features: Features, start: int, end: int):
+ batch_dict = {}
+ for path, dset in h5_obj.items():
+ if path not in features:
+ continue
+ if _is_group(dset):
+ arr = _recursive_load_arrays(dset, features[path], start, end)
+ elif _is_dataset(dset):
+ arr = _load_array(dset, path, start, end)
+ else:
+ raise ValueError(f"Unexpected type {type(dset)}")
+
+ if arr is not None:
+ batch_dict[path] = arr
+
+ if _is_file(h5_obj):
+ return pa.Table.from_pydict(batch_dict)
+
+ if batch_dict:
+ should_chunk, keys, values = False, [], []
+ for k, v in batch_dict.items():
+ if isinstance(v, pa.ChunkedArray):
+ should_chunk = True
+ v = v.combine_chunks()
+ keys.append(k)
+ values.append(v)
+
+ sarr = pa.StructArray.from_arrays(values, names=keys)
+ return pa.chunked_array(sarr) if should_chunk else sarr
+
+
+# ┌─────────────┐
+# │ Utilities │
+# └─────────────┘
+
+
+def _create_sized_feature(dset):
+ dset_shape = dset.shape[1:]
+ value_feature = _np_to_pa_to_hf_value(dset.dtype)
+ return _create_sized_feature_impl(dset_shape, value_feature)
+
+
+def _create_sized_feature_impl(dset_shape, value_feature):
+ dtype_str = value_feature.dtype
+ if any(dim == 0 for dim in dset_shape):
+ logger.warning(
+ f"HDF5 to Arrow: Found a dataset with shape {dset_shape} and dtype {dtype_str} that has a dimension with size 0. Shape information will be lost in the conversion to List({value_feature})."
+ )
+ return List(value_feature)
+
+ rank = len(dset_shape)
+ if rank == 0:
+ return value_feature
+ elif rank == 1:
+ return List(value_feature, length=dset_shape[0])
+ elif rank <= 5:
+ return _sized_arrayxd(rank)(shape=dset_shape, dtype=dtype_str)
+ else:
+ raise TypeError(f"Array{rank}D not supported. Maximum 5 dimensions allowed.")
+
+
+def _sized_arrayxd(rank: int):
+ return {2: Array2D, 3: Array3D, 4: Array4D, 5: Array5D}[rank]
+
+
+def _np_to_pa_to_hf_value(numpy_dtype: np.dtype) -> Value:
+ return Value(dtype=_arrow_to_datasets_dtype(pa.from_numpy_dtype(numpy_dtype)))
+
+
+def _first_dataset(h5_obj, features: Features, prefix=""):
+ for path, dset in h5_obj.items():
+ if path not in features:
+ continue
+ if _is_group(dset):
+ found = _first_dataset(dset, features[path], prefix=f"{prefix}{path}/")
+ if found is not None:
+ return found
+ elif _is_dataset(dset):
+ return f"{prefix}{path}"
+
+
+def _check_dataset_lengths(h5_obj, features: Features) -> int:
+ first_path = _first_dataset(h5_obj, features)
+ if first_path is None:
+ return None
+
+ num_rows = h5_obj[first_path].shape[0]
+ for path, dset in h5_obj.items():
+ if path not in features:
+ continue
+ if _is_dataset(dset):
+ if dset.shape[0] != num_rows:
+ raise ValueError(f"Dataset '{path}' has length {dset.shape[0]} but expected {num_rows}")
+ return num_rows
+
+
+def _is_group(h5_obj) -> bool:
+ import h5py
+
+ return isinstance(h5_obj, h5py.Group) or isinstance(h5_obj, _CompoundGroup)
+
+
+def _is_dataset(h5_obj) -> bool:
+ import h5py
+
+ return isinstance(h5_obj, h5py.Dataset) or isinstance(h5_obj, _CompoundField)
+
+
+def _is_file(h5_obj) -> bool:
+ import h5py
+
+ return isinstance(h5_obj, h5py.File)
+
+
+def _has_zero_dimensions(feature):
+ if isinstance(feature, _ArrayXD):
+ return any(dim == 0 for dim in feature.shape)
+ elif isinstance(feature, List):
+ return feature.length == 0 or _has_zero_dimensions(feature.feature)
+ elif isinstance(feature, LargeList):
+ return _has_zero_dimensions(feature.feature)
+ else:
+ return False
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/iceberg/__init__.py b/venv/Lib/site-packages/datasets/packaged_modules/iceberg/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/iceberg/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/iceberg/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3ae3260cb798f9ce6d6d629d4ba7a128cbd90eb8
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/iceberg/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/iceberg/__pycache__/iceberg.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/iceberg/__pycache__/iceberg.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d0e2f95f98002e0b592d0e273631a49f22dd44d4
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/iceberg/__pycache__/iceberg.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/iceberg/iceberg.py b/venv/Lib/site-packages/datasets/packaged_modules/iceberg/iceberg.py
new file mode 100644
index 0000000000000000000000000000000000000000..447ed10fceb0746154be9977f6d7e3ae5d64afaa
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/iceberg/iceberg.py
@@ -0,0 +1,173 @@
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Dict, List, Optional, Union
+
+import pyarrow as pa
+
+import datasets
+from datasets.builder import Key
+from datasets.features import Features
+from datasets.table import table_cast
+
+
+if TYPE_CHECKING:
+ from pyiceberg.catalog import Catalog
+ from pyiceberg.expressions import BooleanExpression
+ from pyiceberg.table import FileScanTask
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+@dataclass
+class IcebergConfig(datasets.BuilderConfig):
+ """BuilderConfig for Apache Iceberg format.
+
+ Args:
+ catalog (`pyiceberg.catalog.Catalog`):
+ A pre-configured pyiceberg Catalog object.
+ table (`str` or `Dict[str, str]`):
+ Iceberg table identifier, e.g. ``"db.my_table"``.
+ Pass a dict to map split names to table identifiers,
+ e.g. ``{"train": "db.train", "test": "db.test"}``.
+ features (`Features`, *optional*):
+ Cast the data to these features.
+ columns (`List[str]`, *optional*):
+ List of columns to load; others are ignored.
+ filters (`str` or `BooleanExpression`, *optional*):
+ Row filter with predicate pushdown. Accepts a SQL-style string
+ (``"col > 1 AND col2 == 'foo'"``), or a pyiceberg
+ ``BooleanExpression`` object. Parsed by pyiceberg internally.
+ batch_size (`int`, defaults to ``131072``):
+ Number of rows per RecordBatch when reading.
+ snapshot_id (`int`, *optional*):
+ Load a specific snapshot for time-travel queries.
+ """
+
+ catalog: Optional["Catalog"] = None
+ table: Optional[Union[str, Dict[str, str]]] = None
+ features: Optional[datasets.Features] = None
+ columns: Optional[List[str]] = None
+ filters: Optional[Union[str, "BooleanExpression"]] = None
+ batch_size: int = 131072
+ snapshot_id: Optional[int] = None
+
+ def __post_init__(self):
+ super().__post_init__()
+ if self.catalog is None:
+ raise ValueError("`catalog` must be a pyiceberg Catalog object, but got None.")
+ if self.table is None:
+ raise ValueError("`table` must be specified, e.g. table='db.my_table'")
+ # Normalize table to Dict[split_name, table_identifier]
+ if isinstance(self.table, str):
+ self.table = {"train": self.table}
+ # Generate a stable config name for caching
+ if self.name == "default":
+ catalog_id = f"{self.catalog.__class__.__name__}_{self.catalog.name}"
+ table_id = "_".join(sorted(self.table.values()))
+ self.name = f"{catalog_id}_{table_id}"
+
+ def create_config_id(
+ self,
+ config_kwargs: dict,
+ custom_features: Optional[Features] = None,
+ ) -> str:
+ # The catalog object is not picklable (contains SQLAlchemy engines, etc.),
+ # so we replace it with a hashable string representation before the
+ # parent class hashes config_kwargs via dill.
+ config_kwargs = config_kwargs.copy()
+ catalog = config_kwargs.pop("catalog", None)
+ if catalog is not None:
+ config_kwargs["_catalog_id"] = f"{catalog.__class__.__name__}_{catalog.name}"
+ # filters may contain pyiceberg Expression objects that are not picklable
+ filters = config_kwargs.pop("filters", None)
+ if filters is not None:
+ config_kwargs["_filters_repr"] = repr(filters)
+ return super().create_config_id(config_kwargs, custom_features=custom_features)
+
+
+class Iceberg(datasets.ArrowBasedBuilder, datasets.builder._CountableBuilderMixin):
+ BUILDER_CONFIG_CLASS = IcebergConfig
+
+ def _info(self):
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager):
+ splits = []
+ for split_name, table_id in self.config.table.items():
+ iceberg_table = self.config.catalog.load_table(table_id)
+
+ scan_kwargs = {}
+ if self.config.filters is not None:
+ scan_kwargs["row_filter"] = self.config.filters
+ if self.config.columns:
+ scan_kwargs["selected_fields"] = tuple(self.config.columns)
+ if self.config.snapshot_id is not None:
+ scan_kwargs["snapshot_id"] = self.config.snapshot_id
+
+ scan = iceberg_table.scan(**scan_kwargs)
+
+ # Infer features from Arrow schema if not user-provided
+ if self.info.features is None:
+ arrow_schema = scan.projection().as_arrow()
+ self.info.features = datasets.Features.from_arrow_schema(arrow_schema)
+
+ # Plan files for parallel processing: passing a list in gen_kwargs
+ # enables _split_gen_kwargs to distribute tasks across num_proc workers.
+ tasks = list(scan.plan_files())
+
+ # Extract picklable scan context for multiprocessing compatibility.
+ # The scan object itself is not picklable (holds catalog connections),
+ # but these components are individually serializable.
+ scan_context = (
+ scan.table_metadata,
+ scan.io,
+ scan.projection(),
+ scan.row_filter,
+ scan.case_sensitive,
+ scan.limit,
+ )
+
+ splits.append(
+ datasets.SplitGenerator(
+ name=split_name,
+ gen_kwargs={"tasks": tasks, "scan_context": scan_context},
+ )
+ )
+
+ # Drop the catalog reference so the builder becomes picklable for num_proc > 1.
+ # All data needed for reading has been extracted into scan_context above.
+ self.config.catalog = None
+ self.config_kwargs.pop("catalog", None)
+
+ return splits
+
+ def _cast_table(self, pa_table: pa.Table) -> pa.Table:
+ if self.info.features is not None:
+ # More expensive cast to support nested features with keys in a different order
+ # allows str <-> int/float or str to Audio for example
+ pa_table = table_cast(pa_table, self.info.features.arrow_schema)
+ return pa_table
+
+ def _generate_shards(self, tasks: List["FileScanTask"], scan_context):
+ for task in tasks:
+ yield task.file.file_path
+
+ def _generate_num_examples(self, tasks: List["FileScanTask"], scan_context):
+ for task in tasks:
+ yield task.file.record_count
+
+ def _generate_tables(self, tasks: List["FileScanTask"], scan_context):
+ from pyiceberg.io.pyarrow import ArrowScan
+
+ table_metadata, io, projected_schema, row_filter, case_sensitive, limit = scan_context
+ arrow_scan = ArrowScan(
+ table_metadata,
+ io,
+ projected_schema,
+ row_filter,
+ case_sensitive=case_sensitive,
+ limit=limit,
+ )
+ for task_idx, task in enumerate(tasks):
+ for batch_idx, batch in enumerate(arrow_scan.to_record_batches([task])):
+ pa_table = pa.Table.from_batches([batch])
+ yield Key(task_idx, batch_idx), self._cast_table(pa_table)
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/imagefolder/__init__.py b/venv/Lib/site-packages/datasets/packaged_modules/imagefolder/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/imagefolder/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/imagefolder/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4e396caed62d68df9741ead1c2adab96cc12c253
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/imagefolder/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/imagefolder/__pycache__/imagefolder.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/imagefolder/__pycache__/imagefolder.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..fe85aa8b3761318354a7583fdb7c1025afc00a7e
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/imagefolder/__pycache__/imagefolder.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/imagefolder/imagefolder.py b/venv/Lib/site-packages/datasets/packaged_modules/imagefolder/imagefolder.py
new file mode 100644
index 0000000000000000000000000000000000000000..aba9452ad4e12833c60ef78df8b3a0cae519f4a4
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/imagefolder/imagefolder.py
@@ -0,0 +1,103 @@
+import datasets
+
+from ..folder_based_builder import folder_based_builder
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+class ImageFolderConfig(folder_based_builder.FolderBasedBuilderConfig):
+ """BuilderConfig for ImageFolder."""
+
+ drop_labels: bool = None
+ drop_metadata: bool = None
+
+ def __post_init__(self):
+ super().__post_init__()
+
+
+class ImageFolder(folder_based_builder.FolderBasedBuilder):
+ BASE_FEATURE = datasets.Image
+ BASE_COLUMN_NAME = "image"
+ BUILDER_CONFIG_CLASS = ImageFolderConfig
+ EXTENSIONS: list[str] # definition at the bottom of the script
+
+
+# Obtained with:
+# ```
+# import PIL.Image
+# IMAGE_EXTENSIONS = []
+# PIL.Image.init()
+# for ext, format in PIL.Image.EXTENSION.items():
+# if format in PIL.Image.OPEN:
+# IMAGE_EXTENSIONS.append(ext[1:])
+# ```
+# We intentionally do not run this code on launch because:
+# (1) Pillow is an optional dependency, so importing Pillow in global namespace is not allowed
+# (2) To ensure the list of supported extensions is deterministic
+IMAGE_EXTENSIONS = [
+ ".blp",
+ ".bmp",
+ ".dib",
+ ".bufr",
+ ".cur",
+ ".pcx",
+ ".dcx",
+ ".dds",
+ ".ps",
+ ".eps",
+ ".fit",
+ ".fits",
+ ".fli",
+ ".flc",
+ ".ftc",
+ ".ftu",
+ ".gbr",
+ ".gif",
+ ".grib",
+ # ".h5", # may contain zero or several images
+ # ".hdf", # may contain zero or several images
+ ".png",
+ ".apng",
+ ".jp2",
+ ".j2k",
+ ".jpc",
+ ".jpf",
+ ".jpx",
+ ".j2c",
+ ".icns",
+ ".ico",
+ ".im",
+ ".iim",
+ ".tif",
+ ".tiff",
+ ".jfif",
+ ".jpe",
+ ".jpg",
+ ".jpeg",
+ ".mpg",
+ ".mpeg",
+ ".msp",
+ ".pcd",
+ ".pxr",
+ ".pbm",
+ ".pgm",
+ ".ppm",
+ ".pnm",
+ ".psd",
+ ".bw",
+ ".rgb",
+ ".rgba",
+ ".sgi",
+ ".ras",
+ ".tga",
+ ".icb",
+ ".vda",
+ ".vst",
+ ".webp",
+ ".wmf",
+ ".emf",
+ ".xbm",
+ ".xpm",
+]
+ImageFolder.EXTENSIONS = IMAGE_EXTENSIONS
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/json/__init__.py b/venv/Lib/site-packages/datasets/packaged_modules/json/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/json/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/json/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e4a0691888eff47dd4c388c5a32fa93e27e09149
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/json/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/json/__pycache__/json.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/json/__pycache__/json.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d32d81e37feb41698433fb7b86907e09fc78be0e
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/json/__pycache__/json.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/json/json.py b/venv/Lib/site-packages/datasets/packaged_modules/json/json.py
new file mode 100644
index 0000000000000000000000000000000000000000..393d33e81316cb47fa30f530d88a1558f53cba2c
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/json/json.py
@@ -0,0 +1,489 @@
+import io
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal, Optional
+
+import pandas as pd
+import pyarrow as pa
+import pyarrow.json as paj
+
+import datasets
+import datasets.config
+from datasets import List, Value
+from datasets.builder import Key
+from datasets.table import table_cast
+from datasets.utils.file_utils import readline
+from datasets.utils.json import (
+ find_mixed_struct_types_field_paths,
+ get_json_field_path_from_pyarrow_json_error,
+ get_json_field_paths_from_feature,
+ insert_json_field_path,
+ json_encode_field,
+ json_encode_fields_in_json_lines,
+ set_json_types_in_feature,
+ ujson_dumps,
+ ujson_loads,
+)
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+def pandas_read_json(path_or_buf, **kwargs):
+ if datasets.config.PANDAS_VERSION.major >= 2:
+ kwargs["dtype_backend"] = "pyarrow"
+ return pd.read_json(path_or_buf, **kwargs)
+
+
+class FullReadDisallowed(Exception):
+ pass
+
+
+@dataclass
+class JsonConfig(datasets.BuilderConfig):
+ """BuilderConfig for JSON."""
+
+ features: Optional[datasets.Features] = None
+ encoding: str = "utf-8"
+ encoding_errors: Optional[str] = None
+ field: Optional[str] = None
+ use_threads: bool = True # deprecated
+ block_size: Optional[int] = None # deprecated
+ chunksize: int = 10 << 20 # 10MB
+ newlines_in_values: Optional[bool] = None
+ on_mixed_types: Optional[Literal["use_json"]] = "use_json"
+ parse_agent_traces: bool = True
+
+ def __post_init__(self):
+ super().__post_init__()
+
+
+class Json(datasets.ArrowBasedBuilder):
+ BUILDER_CONFIG_CLASS = JsonConfig
+
+ def _info(self):
+ if self.config.block_size is not None:
+ logger.warning("The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead")
+ self.config.chunksize = self.config.block_size
+ if self.config.use_threads is not True:
+ logger.warning(
+ "The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore."
+ )
+ if self.config.newlines_in_values is not None:
+ raise ValueError("The JSON loader parameter `newlines_in_values` is no longer supported")
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager):
+ """We handle string, list and dicts in datafiles"""
+ if not self.config.data_files:
+ raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+ dl_manager.download_config.extract_on_the_fly = True
+ base_data_files = dl_manager.download(self.config.data_files)
+ extracted_data_files = dl_manager.extract(base_data_files)
+ splits = []
+ for split_name, extracted_files in extracted_data_files.items():
+ files_iterables = [dl_manager.iter_files(extracted_file) for extracted_file in extracted_files]
+ splits.append(
+ datasets.SplitGenerator(
+ name=split_name,
+ gen_kwargs={
+ "files_iterables": files_iterables,
+ "base_files": base_data_files[split_name],
+ "original_files": self.config.data_files[split_name],
+ },
+ )
+ )
+ if self.info.features is None:
+ try:
+ pa_table = next(iter(self._generate_tables(**splits[0].gen_kwargs, allow_full_read=False)))[1]
+ self.info.features = datasets.Features.from_arrow_schema(pa_table.schema)
+ if self.config.parse_agent_traces and has_agent_traces_markers(self.info.features):
+ self.info.features = AGENT_TRACES_FEATURES
+ except FullReadDisallowed:
+ pass
+ return splits
+
+ def _cast_table(self, pa_table: pa.Table, json_field_paths=()) -> pa.Table:
+ if self.info.features is not None:
+ # adding missing columns
+ for column_name in set(self.info.features) - set(pa_table.column_names):
+ type = self.info.features.arrow_schema.field(column_name).type
+ pa_table = pa_table.append_column(column_name, pa.array([None] * len(pa_table), type=type))
+ # convert to string when needed
+ for i, column_name in enumerate(pa_table.column_names):
+ if pa.types.is_struct(pa_table[column_name].type) and self.info.features.get(
+ column_name, None
+ ) == Value("string"):
+ jsonl = (
+ pa_table[column_name]
+ .to_pandas(types_mapper=pd.ArrowDtype)
+ .to_json(orient="records", lines=True)
+ )
+ string_array = pa.array(
+ (None if x.strip() == "null" else x.strip() for x in jsonl.split("\n") if x.strip()),
+ type=pa.string(),
+ )
+ pa_table = pa_table.set_column(i, column_name, string_array)
+ # more expensive cast to support nested structures with keys in a different order
+ # allows str <-> int/float or str to Audio for example
+ pa_table = table_cast(pa_table, self.info.features.arrow_schema)
+ elif json_field_paths:
+ features = datasets.Features.from_arrow_schema(pa_table.schema)
+ features = set_json_types_in_feature(features, json_field_paths)
+ pa_table = table_cast(pa_table, features.arrow_schema)
+ return pa_table
+
+ def _generate_shards(self, base_files, files_iterables, original_files):
+ yield from base_files
+
+ def _generate_tables(self, base_files, files_iterables, original_files, allow_full_read=True):
+ json_field_paths = []
+ is_agent_traces = False
+
+ if self.info.features is not None:
+ if self.info.features == AGENT_TRACES_FEATURES:
+ is_agent_traces = True
+ if datasets.config.TEICH_AVAILABLE:
+ from teich import convert_trace_to_training_example
+ else:
+ raise ImportError("To support decoding agent traces, please install 'teich'.")
+ json_field_paths = get_json_field_paths_from_feature(self.info.features)
+
+ for shard_idx, files_iterable in enumerate(files_iterables):
+ for file in files_iterable:
+ # If the file is one json object and if we need to look at the items in one specific field
+ if self.config.field is not None:
+ if not allow_full_read:
+ raise FullReadDisallowed()
+ with open(file, encoding=self.config.encoding, errors=self.config.encoding_errors) as f:
+ dataset = ujson_loads(f.read())
+ # We keep only the field we are interested in
+ dataset = dataset[self.config.field]
+ df = pandas_read_json(io.StringIO(ujson_dumps(dataset)))
+ if df.columns.tolist() == [0]:
+ df.columns = list(self.config.features) if self.config.features else ["text"]
+ pa_table = pa.Table.from_pandas(df, preserve_index=False)
+ yield Key(shard_idx, 0), self._cast_table(pa_table)
+
+ # If the files are agent traces (one row = one file)
+ elif is_agent_traces:
+ trace_file = Path(file)
+ trace_events = trace_file.read_text(encoding="utf-8").splitlines()
+ harness, session_id, prompt, sent_at, num_user_messages, num_tool_calls = parse_traces_info(
+ trace_events
+ )
+ file_path = original_files[shard_idx]
+ if self.base_path is not None and file_path.startswith(self.base_path):
+ file_path = os.path.relpath(file_path, self.base_path)
+ training_example = convert_trace_to_training_example(trace_file)
+ example = {
+ **dict.fromkeys(AGENT_TRACES_FEATURES),
+ **training_example.to_dict(),
+ "harness": harness,
+ "session_id": session_id,
+ "prompt": prompt,
+ "sent_at": sent_at,
+ "num_user_messages": num_user_messages,
+ "num_tool_calls": num_tool_calls,
+ "trace": trace_events,
+ "file_path": file_path,
+ }
+ for json_field_path in json_field_paths:
+ example = json_encode_field(example, json_field_path)
+ pa_table = pa.Table.from_pylist([example])
+ yield Key(shard_idx, 0), self._cast_table(pa_table)
+
+ # If the file has one json object per line
+ else:
+ with open(file, "rb") as f:
+ batch_idx = 0
+ # Use block_size equal to the chunk size divided by 32 to leverage multithreading
+ # Set a default minimum value of 16kB if the chunk size is really small
+ block_size = max(self.config.chunksize // 32, 16 << 10)
+ encoding_errors = (
+ self.config.encoding_errors if self.config.encoding_errors is not None else "strict"
+ )
+ while True:
+ batch = f.read(self.config.chunksize)
+ if not batch:
+ break
+ if batch.startswith(b"["):
+ if not allow_full_read:
+ raise FullReadDisallowed()
+ else:
+ # convert to JSON Lines
+ full_data = batch + f.read()
+ if b"{" in batch[:100].split(b'"', 1)[0]: # list of objects
+ batch = "\n".join(ujson_dumps(x) for x in ujson_loads(full_data)).encode()
+ else: # list of strings
+ batch = "\n".join(
+ ujson_dumps({"text": x}) for x in ujson_loads(full_data)
+ ).encode()
+ # Finish current line
+ try:
+ batch += f.readline()
+ except (AttributeError, io.UnsupportedOperation):
+ batch += readline(f)
+ # PyArrow only accepts utf-8 encoded bytes
+ if self.config.encoding != "utf-8":
+ batch = batch.decode(self.config.encoding, errors=encoding_errors).encode("utf-8")
+ # On first batch we check for lists of objects with arbitrary fields
+ if (
+ shard_idx == 0
+ and batch_idx == 0
+ and self.info.features is None
+ and self.config.on_mixed_types == "use_json"
+ ):
+ try:
+ examples = [ujson_loads(line) for line in batch.splitlines()]
+ except ValueError:
+ # the file is likely not JSON Lines and may contain one single multi-line JSON object
+ pass
+ else:
+ json_field_paths += find_mixed_struct_types_field_paths(examples)
+ # Re-encode JSON fields
+ original_batch = batch
+ if json_field_paths:
+ examples = [ujson_loads(line) for line in batch.splitlines()]
+ for json_field_path in json_field_paths:
+ examples = [json_encode_field(example, json_field_path) for example in examples]
+ batch = "\n".join(ujson_dumps(example) for example in examples).encode()
+ # Disable parallelism if block size is ~ len(batch) to avoid segfault
+ block_size = len(batch) if len(batch) // 8 > block_size else block_size
+ try:
+ while True:
+ try:
+ pa_table = paj.read_json(
+ io.BytesIO(batch), read_options=paj.ReadOptions(block_size=block_size)
+ )
+ break
+ except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
+ if batch.startswith(b"["): # paj.read_json only supports json lines
+ raise
+ elif self.config.on_mixed_types == "use_json" and (
+ isinstance(e, pa.ArrowInvalid)
+ and "JSON parse error: Column(" in str(e)
+ and ") changed from" in str(e)
+ ):
+ json_field_path = get_json_field_path_from_pyarrow_json_error(str(e))
+ insert_json_field_path(json_field_paths, json_field_path)
+ batch = json_encode_fields_in_json_lines(original_batch, json_field_paths)
+ elif (
+ "straddling" in str(e) or "JSON conversion to" in str(e)
+ ) and block_size < len(batch):
+ # Increase the block size in case it was too small.
+ # The block size will be reset for the next file.
+ # this is needed in case of "stradding" or for some JSON conversions (see https://github.com/huggingface/datasets/issues/2799)
+ logger.debug(
+ f"Batch of {len(batch)} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."
+ )
+ block_size *= 2
+ else:
+ raise
+ except pa.ArrowInvalid as e:
+ if not allow_full_read:
+ raise FullReadDisallowed()
+ try:
+ with open(
+ file, encoding=self.config.encoding, errors=self.config.encoding_errors
+ ) as f:
+ df = pandas_read_json(f)
+ except ValueError:
+ logger.error(f"Failed to load JSON from file '{file}' with error {type(e)}: {e}")
+ raise e
+ if df.columns.tolist() == [0]:
+ df.columns = list(self.config.features) if self.config.features else ["text"]
+ try:
+ pa_table = pa.Table.from_pandas(df, preserve_index=False)
+ except pa.ArrowInvalid as e:
+ logger.error(
+ f"Failed to convert pandas DataFrame to Arrow Table from file '{file}' with error {type(e)}: {e}"
+ )
+ raise ValueError(
+ f"Failed to convert pandas DataFrame to Arrow Table from file {file}."
+ ) from None
+ yield Key(shard_idx, 0), self._cast_table(pa_table)
+ break
+ yield (
+ Key(shard_idx, batch_idx),
+ self._cast_table(pa_table, json_field_paths=json_field_paths),
+ )
+ batch_idx += 1
+
+
+AGENT_TRACES_TYPES_VALUES = {
+ "claude_code": ["user", "assistant", "system"],
+ "pi": ["session", "message"],
+ "codex": ["session_meta", "turn_context", "response_item", "event_msg"],
+}
+AGENT_TRACES_TYPE_TO_HARNESS = {}
+for _harness, _trace_types in AGENT_TRACES_TYPES_VALUES.items():
+ for _trace_type in _trace_types:
+ AGENT_TRACES_TYPE_TO_HARNESS[_trace_type] = _harness
+
+
+AGENT_TRACES_FEATURES_MARKERS = {
+ "claude_code": datasets.Features(
+ {
+ "type": Value("string"),
+ "message": datasets.Json(),
+ }
+ ),
+ "pi": datasets.Features(
+ {
+ "type": Value("string"),
+ "message": datasets.Json(),
+ }
+ ),
+ "codex": datasets.Features(
+ {
+ "type": Value("string"),
+ "payload": datasets.Json(),
+ }
+ ),
+}
+
+AGENT_TRACES_FEATURES = datasets.Features(
+ {
+ # basic features
+ "harness": Value("string"),
+ "session_id": Value("string"),
+ # teich features
+ "prompt": Value("string"),
+ "messages": List(datasets.Json()),
+ "tools": List(datasets.Json()),
+ "metadata": datasets.Json(),
+ # bonus features
+ "sent_at": Value("string"),
+ "num_user_messages": Value("int64"),
+ "num_tool_calls": Value("int64"),
+ "trace": List(datasets.Json()),
+ "file_path": Value("string"),
+ }
+)
+
+
+def has_agent_traces_markers(features: datasets.Features) -> bool:
+ for agent_traces_features_marker in AGENT_TRACES_FEATURES_MARKERS.values():
+ if all(features.get(key) == feature for key, feature in agent_traces_features_marker.items()):
+ return True
+ return False
+
+
+def parse_traces_info(
+ trace_events: list[str],
+) -> tuple[Optional[str], Optional[str], Optional[str], Optional[str], int, int]:
+ harness, session_id, prompt, sent_at = None, None, None, None
+ # prompt/sent_at describe the first user message; the counters summarize the whole trace file.
+ # Codex response_item user messages can include context files (for example AGENTS.md), so only event_msg
+ # user_message records are treated as Codex user messages.
+ num_user_messages = 0
+ num_tool_calls = 0
+ for event in trace_events:
+ decoded_event = ujson_loads(event)
+ if harness is None:
+ if "type" in decoded_event and isinstance(decoded_event["type"], str):
+ harness = AGENT_TRACES_TYPE_TO_HARNESS.get(decoded_event["type"])
+ if session_id is None:
+ session_id = get_session_id(decoded_event)
+ if (
+ session_id is not None
+ and decoded_event.get("type") == "session"
+ and isinstance(decoded_event.get("cwd"), str)
+ and "/.openclaw/" in decoded_event["cwd"]
+ ):
+ harness = "openclaw"
+ user_prompt = get_user_prompt(decoded_event)
+ if user_prompt is not None:
+ num_user_messages += 1
+ if prompt is None:
+ prompt = user_prompt
+ sent_at = get_trace_event_timestamp(decoded_event)
+ num_tool_calls += get_tool_call_count(decoded_event)
+ return harness, session_id, prompt, sent_at, num_user_messages, num_tool_calls
+
+
+def get_session_id(trace: dict) -> Optional[str]:
+ # claude
+ if isinstance(trace.get("sessionId"), str):
+ return trace["sessionId"]
+ # claude (not sure but this format does exist online)
+ if isinstance(trace.get("session_id"), str):
+ return trace["session_id"]
+ # codex
+ if isinstance(trace.get("payload"), dict) and isinstance(trace["payload"].get("id"), str):
+ return trace["payload"]["id"]
+ # pi / openclaw (openclaw embeds pi-agent; distinguish via cwd)
+ if trace.get("type") == "session" and isinstance(trace.get("id"), str):
+ return trace["id"]
+ return None
+
+
+def get_user_prompt(trace_event: dict) -> Optional[str]:
+ if trace_event.get("type") == "user" and isinstance(trace_event.get("message"), dict):
+ message = trace_event["message"]
+ if message.get("role") == "user":
+ return get_content_text(message.get("content"))
+
+ if trace_event.get("type") == "message":
+ if isinstance(trace_event.get("message"), dict):
+ message = trace_event["message"]
+ if message.get("role") == "user":
+ return get_content_text(message.get("content"))
+ if trace_event.get("role") == "user":
+ return get_content_text(trace_event.get("content"))
+
+ if trace_event.get("type") == "event_msg" and isinstance(trace_event.get("payload"), dict):
+ payload = trace_event["payload"]
+ if payload.get("type") == "user_message":
+ return get_content_text(payload.get("message"))
+
+ return None
+
+
+def get_tool_call_count(trace_event: dict) -> int:
+ trace_type = trace_event.get("type")
+ if trace_type == "response_item":
+ payload = trace_event.get("payload")
+ if isinstance(payload, dict) and payload.get("type") == "function_call":
+ return 1
+ return 0
+
+ if trace_type not in {"assistant", "message"} or not isinstance(trace_event.get("message"), dict):
+ return 0
+ message = trace_event["message"]
+ if message.get("role") != "assistant":
+ return 0
+ content = message.get("content")
+ if not isinstance(content, list):
+ return 0
+ return sum(
+ 1
+ for content_part in content
+ if isinstance(content_part, dict) and content_part.get("type") in {"tool_use", "toolCall"}
+ )
+
+
+def get_trace_event_timestamp(trace_event: dict) -> Optional[str]:
+ timestamp = trace_event.get("timestamp")
+ if isinstance(timestamp, str):
+ return timestamp
+ return None
+
+
+def get_content_text(content) -> Optional[str]:
+ if isinstance(content, str):
+ return content
+ if isinstance(content, list):
+ content_parts = []
+ for content_part in content:
+ if isinstance(content_part, str):
+ content_parts.append(content_part)
+ elif isinstance(content_part, dict):
+ text = content_part.get("text")
+ if isinstance(text, str):
+ content_parts.append(text)
+ if content_parts:
+ return "\n".join(content_parts)
+ return None
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/lance/__init__.py b/venv/Lib/site-packages/datasets/packaged_modules/lance/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/lance/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/lance/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..71ff965f7a77897f72d3202bd2c48ee0b17c88a1
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/lance/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/lance/__pycache__/lance.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/lance/__pycache__/lance.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..60b020b062c5d7280c0e0df25f825a0b8ebca1da
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/lance/__pycache__/lance.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/lance/lance.py b/venv/Lib/site-packages/datasets/packaged_modules/lance/lance.py
new file mode 100644
index 0000000000000000000000000000000000000000..017458ec94e477ee07f6735fe9bac57743e7cc6b
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/lance/lance.py
@@ -0,0 +1,236 @@
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Dict, List, Optional
+
+import pyarrow as pa
+from huggingface_hub import HfApi
+
+import datasets
+from datasets import Audio, Image, Video
+from datasets.builder import Key
+from datasets.table import table_cast
+from datasets.utils.file_utils import is_local_path
+
+
+if TYPE_CHECKING:
+ import lance
+ import lance.file
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+MAGIC_BYTES_EXTENSION_AND_FEATURE_TYPES = [
+ ("1A 45 DF A3", ".mkv", Video()),
+ ("66 74 79 70 69 73 6F 6D", ".mp4", Video()),
+ ("66 74 79 70 4D 53 4E 56", ".mp4", Video()),
+ ("52 49 46 46", ".avi", Video()),
+ ("00 00 01 BA", ".mpeg", Video()),
+ ("00 00 01 BA", ".mpeg", Video()),
+ ("00 00 01 B3", ".mov", Video()),
+ ("89 50 4E 47", ".png", Image()),
+ ("FF D8", ".jpg", Image()),
+ ("49 49", ".tif", Image()),
+ ("47 49 46 38", ".gif", Image()),
+ ("52 49 46 46", ".wav", Audio()),
+ ("49 44 33", ".mp3", Audio()),
+ ("66 4C 61 43", ".flac", Audio()),
+]
+
+
+@dataclass
+class LanceConfig(datasets.BuilderConfig):
+ """
+ BuilderConfig for Lance format.
+
+ Args:
+ features: (`Features`, *optional*):
+ Cast the data to `features`.
+ columns: (`List[str]`, *optional*):
+ List of columns to load, the other ones are ignored.
+ batch_size: (`int`, *optional*):
+ Size of the RecordBatches to iterate on. Default to 256.
+ token: (`str`, *optional*):
+ Optional HF token to use to download datasets.
+ """
+
+ features: Optional[datasets.Features] = None
+ columns: Optional[List[str]] = None
+ batch_size: Optional[int] = 256
+ token: Optional[str] = None
+
+
+def resolve_dataset_uris(files: List[str]) -> Dict[str, List[str]]:
+ dataset_uris = set()
+ for file_path in files:
+ path = Path(file_path)
+ if path.parent.name in {"_transactions", "_indices", "_versions"}:
+ dataset_root = path.parent.parent
+ dataset_uris.add(str(dataset_root))
+ return list(dataset_uris)
+
+
+def _fix_hf_uri(uri: str) -> str:
+ # replace the revision tag from hf uri
+ if "@" in uri:
+ matched = re.match(r"(hf://.+?)(@[0-9a-f]+)(/.*)", uri)
+ if matched:
+ uri = matched.group(1) + matched.group(3)
+ return uri
+
+
+def _fix_local_version_file(uri: str) -> str:
+ # replace symlinks with real files for _version
+ if "/_versions/" in uri and is_local_path(uri):
+ path = Path(uri)
+ if path.is_symlink():
+ data = path.read_bytes()
+ path.unlink()
+ path.write_bytes(data)
+ return uri
+
+
+class Lance(datasets.ArrowBasedBuilder, datasets.builder._CountableBuilderMixin):
+ BUILDER_CONFIG_CLASS = LanceConfig
+ METADATA_EXTENSIONS = [".idx", ".txn", ".manifest"]
+ METADATA_FILE_NAMES = ["latest_version_hint.json"]
+
+ def _info(self):
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager):
+ import lance
+ import lance.file
+
+ if not self.config.data_files:
+ raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+ if self.repo_id:
+ api = HfApi(**dl_manager.download_config.storage_options.get("hf", {}))
+ dataset_sha = api.dataset_info(self.repo_id).sha
+ if dataset_sha != self.hash:
+ raise NotImplementedError(
+ f"lance doesn't support loading other revisions than 'main' yet, but got {self.hash}"
+ )
+ data_files = dl_manager.download(self.config.data_files)
+
+ # TODO: remove once Lance supports HF links with revisions
+ data_files = {split: [_fix_hf_uri(file) for file in files] for split, files in data_files.items()}
+ # TODO: remove once Lance supports symlinks for _version files
+ data_files = {split: [_fix_local_version_file(file) for file in files] for split, files in data_files.items()}
+
+ splits: list[datasets.SplitGenerator] = []
+ for split_name, files in data_files.items():
+ storage_options = dl_manager.download_config.storage_options.get(files[0].split("://", 1)[0])
+
+ lance_dataset_uris = resolve_dataset_uris(files)
+ if lance_dataset_uris:
+ lance_datasets = [lance.dataset(uri, storage_options=storage_options) for uri in lance_dataset_uris]
+ fragments = [frag for lance_dataset in lance_datasets for frag in lance_dataset.get_fragments()]
+ if self.info.features is None:
+ pa_schema = fragments[0]._ds.schema
+ first_row_first_bytes = {}
+ for field in pa_schema:
+ if self.config.columns is not None and field.name not in self.config.columns:
+ continue
+ if pa.types.is_binary(field.type) or pa.types.is_large_binary(field.type):
+ try:
+ first_row_first_bytes[field.name] = (
+ lance_datasets[0].take_blobs(field.name, [0])[0].read(16)
+ )
+ except ValueError:
+ first_row_first_bytes[field.name] = (
+ lance_datasets[0].take([0], [field.name]).to_pylist()[0][field.name][:16]
+ )
+ splits.append(
+ datasets.SplitGenerator(
+ name=split_name,
+ gen_kwargs={"fragments": fragments, "lance_files_paths": None, "lance_files": None},
+ )
+ )
+ else:
+ lance_files = [
+ lance.file.LanceFileReader(file, storage_options=storage_options, columns=self.config.columns)
+ for file in files
+ ]
+ if self.info.features is None:
+ pa_schema = lance_files[0].metadata().schema
+ first_row_first_bytes = {
+ field_name: value[:16]
+ for field_name, value in lance_files[0].take_rows([0]).to_table().to_pylist()[0].items()
+ if isinstance(value, bytes)
+ }
+ splits.append(
+ datasets.SplitGenerator(
+ name=split_name,
+ gen_kwargs={"fragments": None, "lance_files_paths": files, "lance_files": lance_files},
+ )
+ )
+ if self.info.features is None:
+ if self.config.columns:
+ fields = [
+ pa_schema.field(name) for name in self.config.columns if pa_schema.get_field_index(name) != -1
+ ]
+ pa_schema = pa.schema(fields)
+ features = datasets.Features.from_arrow_schema(pa_schema)
+ for field_name, first_bytes in first_row_first_bytes.items():
+ for magic_bytes_hex, _, feature_type in MAGIC_BYTES_EXTENSION_AND_FEATURE_TYPES:
+ magic_bytes = bytes.fromhex(magic_bytes_hex)
+ if magic_bytes in first_bytes[: len(magic_bytes) * 2]: # allow some padding
+ features[field_name] = feature_type
+ break
+ self.info.features = features
+
+ return splits
+
+ def _cast_table(self, pa_table: pa.Table) -> pa.Table:
+ if self.info.features is not None:
+ # more expensive cast to support nested features with keys in a different order
+ # allows str <-> int/float or str to Audio for example
+ pa_table = table_cast(pa_table, self.info.features.arrow_schema)
+ return pa_table
+
+ def _generate_shards(
+ self,
+ fragments: Optional[List["lance.LanceFragment"]],
+ lance_files_paths: Optional[list[str]],
+ lance_files: Optional[List["lance.file.LanceFileReader"]],
+ ):
+ if fragments:
+ for fragment in fragments:
+ paths = [data_file.path for data_file in fragment.metadata.data_files()]
+ yield paths[0] if len(paths) == 1 else {"fragment_data_files": paths}
+ else:
+ yield from lance_files_paths
+
+ def _generate_num_examples(
+ self,
+ fragments: Optional[List["lance.LanceFragment"]],
+ lance_files_paths: Optional[list[str]],
+ lance_files: Optional[List["lance.file.LanceFileReader"]],
+ ):
+ if fragments:
+ for fragment in fragments:
+ yield fragment.count_rows()
+ else:
+ for lance_file in lance_files:
+ yield lance_file.num_rows()
+
+ def _generate_tables(
+ self,
+ fragments: Optional[List["lance.LanceFragment"]],
+ lance_files_paths: Optional[list[str]],
+ lance_files: Optional[List["lance.file.LanceFileReader"]],
+ ):
+ if fragments:
+ for frag_idx, fragment in enumerate(fragments):
+ for batch_idx, batch in enumerate(
+ fragment.to_batches(
+ columns=self.config.columns, batch_size=self.config.batch_size, blob_handling="all_binary"
+ )
+ ):
+ table = pa.Table.from_batches([batch])
+ yield Key(frag_idx, batch_idx), self._cast_table(table)
+ else:
+ for file_idx, lance_file in enumerate(lance_files):
+ for batch_idx, batch in enumerate(lance_file.read_all(batch_size=self.config.batch_size).to_batches()):
+ table = pa.Table.from_batches([batch])
+ yield Key(file_idx, batch_idx), self._cast_table(table)
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/meshfolder/__init__.py b/venv/Lib/site-packages/datasets/packaged_modules/meshfolder/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/meshfolder/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/meshfolder/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6ea645c76861aec49323a8d37461dd56c06fc98b
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/meshfolder/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/meshfolder/__pycache__/meshfolder.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/meshfolder/__pycache__/meshfolder.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f2134052b9f40f3654d56b58ffa71456a535d82d
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/meshfolder/__pycache__/meshfolder.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/meshfolder/meshfolder.py b/venv/Lib/site-packages/datasets/packaged_modules/meshfolder/meshfolder.py
new file mode 100644
index 0000000000000000000000000000000000000000..293c66f62e101b20d9e3ceb7bf3c00fc49ceabfc
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/meshfolder/meshfolder.py
@@ -0,0 +1,31 @@
+import datasets
+
+from ..folder_based_builder import folder_based_builder
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+class MeshFolderConfig(folder_based_builder.FolderBasedBuilderConfig):
+ """BuilderConfig for MeshFolder."""
+
+ drop_labels: bool = None
+ drop_metadata: bool = None
+
+ def __post_init__(self):
+ super().__post_init__()
+
+
+class MeshFolder(folder_based_builder.FolderBasedBuilder):
+ BASE_FEATURE = datasets.Mesh
+ BASE_COLUMN_NAME = "mesh"
+ BUILDER_CONFIG_CLASS = MeshFolderConfig
+ EXTENSIONS: list[str] # definition at the bottom of the script
+
+
+MESH_EXTENSIONS = [
+ ".glb",
+ ".ply",
+ ".stl",
+]
+MeshFolder.EXTENSIONS = MESH_EXTENSIONS
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/niftifolder/__init__.py b/venv/Lib/site-packages/datasets/packaged_modules/niftifolder/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/niftifolder/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/niftifolder/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..12d27776a739fc2e11d25d62406103e81533e213
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/niftifolder/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/niftifolder/__pycache__/niftifolder.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/niftifolder/__pycache__/niftifolder.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0fa988eee6fa3bc0248d2be0cfc51e9579ee9885
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/niftifolder/__pycache__/niftifolder.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/niftifolder/niftifolder.py b/venv/Lib/site-packages/datasets/packaged_modules/niftifolder/niftifolder.py
new file mode 100644
index 0000000000000000000000000000000000000000..9c9dea077c3bd0abe0d81e73b48b9ef118ae1179
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/niftifolder/niftifolder.py
@@ -0,0 +1,23 @@
+import datasets
+
+from ..folder_based_builder import folder_based_builder
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+class NiftiFolderConfig(folder_based_builder.FolderBasedBuilderConfig):
+ """BuilderConfig for NiftiFolder."""
+
+ drop_labels: bool = None
+ drop_metadata: bool = None
+
+ def __post_init__(self):
+ super().__post_init__()
+
+
+class NiftiFolder(folder_based_builder.FolderBasedBuilder):
+ BASE_FEATURE = datasets.Nifti
+ BASE_COLUMN_NAME = "nifti"
+ BUILDER_CONFIG_CLASS = NiftiFolderConfig
+ EXTENSIONS: list[str] = [".nii"]
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/pandas/__init__.py b/venv/Lib/site-packages/datasets/packaged_modules/pandas/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/pandas/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/pandas/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..a0409ef414e0ec79eb036dea67b8d24a2331bcbd
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/pandas/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/pandas/__pycache__/pandas.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/pandas/__pycache__/pandas.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7934fbe6072f02d6ac90705831d0c9e1dffd5bc1
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/pandas/__pycache__/pandas.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/pandas/pandas.py b/venv/Lib/site-packages/datasets/packaged_modules/pandas/pandas.py
new file mode 100644
index 0000000000000000000000000000000000000000..13577949f0621781600f8eb88b67b6cab1527eb3
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/pandas/pandas.py
@@ -0,0 +1,57 @@
+import warnings
+from dataclasses import dataclass
+from typing import Optional
+
+import pandas as pd
+import pyarrow as pa
+
+import datasets
+from datasets.builder import Key
+from datasets.table import table_cast
+
+
+@dataclass
+class PandasConfig(datasets.BuilderConfig):
+ """BuilderConfig for Pandas."""
+
+ features: Optional[datasets.Features] = None
+
+ def __post_init__(self):
+ super().__post_init__()
+
+
+class Pandas(datasets.ArrowBasedBuilder):
+ BUILDER_CONFIG_CLASS = PandasConfig
+
+ def _info(self):
+ warnings.warn(
+ "The Pandas builder is deprecated and will be removed in the next major version of datasets.",
+ FutureWarning,
+ )
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager):
+ """We handle string, list and dicts in datafiles"""
+ if not self.config.data_files:
+ raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+ data_files = dl_manager.download(self.config.data_files)
+ splits = []
+ for split_name, files in data_files.items():
+ splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files}))
+ return splits
+
+ def _cast_table(self, pa_table: pa.Table) -> pa.Table:
+ if self.config.features is not None:
+ # more expensive cast to support nested features with keys in a different order
+ # allows str <-> int/float or str to Audio for example
+ pa_table = table_cast(pa_table, self.config.features.arrow_schema)
+ return pa_table
+
+ def _generate_shards(self, files):
+ yield from files
+
+ def _generate_tables(self, files):
+ for i, file in enumerate(files):
+ with open(file, "rb") as f:
+ pa_table = pa.Table.from_pandas(pd.read_pickle(f))
+ yield Key(i, 0), self._cast_table(pa_table)
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/parquet/__init__.py b/venv/Lib/site-packages/datasets/packaged_modules/parquet/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/parquet/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/parquet/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..90ef2572ce9df89b11891b1d100a6f77e937955b
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/parquet/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/parquet/__pycache__/parquet.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/parquet/__pycache__/parquet.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c5d319aaf7e67acb488e96bc1a99051467d2293f
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/parquet/__pycache__/parquet.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/parquet/parquet.py b/venv/Lib/site-packages/datasets/packaged_modules/parquet/parquet.py
new file mode 100644
index 0000000000000000000000000000000000000000..d05ba18a298100c1254660eaf2606d4adbc9842f
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/parquet/parquet.py
@@ -0,0 +1,234 @@
+import gc
+from dataclasses import dataclass
+from typing import Literal, Optional, Union
+
+import pyarrow as pa
+import pyarrow.dataset as ds
+import pyarrow.parquet as pq
+from packaging import version
+
+import datasets
+import datasets.config
+from datasets.builder import Key
+from datasets.table import table_cast
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+@dataclass
+class ParquetConfig(datasets.BuilderConfig):
+ """
+ BuilderConfig for Parquet.
+
+ Args:
+ batch_size (`int`, *optional*):
+ Size of the RecordBatches to iterate on.
+ The default is the row group size (defined by the first row group).
+ columns (`list[str]`, *optional*)
+ List of columns to load, the other ones are ignored.
+ All columns are loaded by default.
+ features: (`Features`, *optional*):
+ Cast the data to `features`.
+ filters (`Union[pyarrow.dataset.Expression, list[tuple], list[list[tuple]]]`, *optional*):
+ Return only the rows matching the filter.
+ If possible the predicate will be pushed down to exploit the partition information
+ or internal metadata found in the data source, e.g. Parquet statistics.
+ Otherwise filters the loaded RecordBatches before yielding them.
+ fragment_scan_options (`pyarrow.dataset.ParquetFragmentScanOptions`, *optional*)
+ Scan-specific options for Parquet fragments.
+ This is especially useful to configure buffering and caching.
+
+
+ on_bad_files (`Literal["error", "warn", "skip"]`, *optional*, defaults to "error")
+ Specify what to do upon encountering a bad file (a file that can't be read). Allowed values are :
+ * 'error', raise an Exception when a bad file is encountered.
+ * 'warn', raise a warning when a bad file is encountered and skip that file.
+ * 'skip', skip bad files without raising or warning when they are encountered.
+
+
+
+ Example:
+
+ Load a subset of columns:
+
+ ```python
+ >>> ds = load_dataset(parquet_dataset_id, columns=["col_0", "col_1"])
+ ```
+
+ Stream data and efficiently filter data, possibly skipping entire files or row groups:
+
+ ```python
+ >>> filters = [("col_0", "==", 0)]
+ >>> ds = load_dataset(parquet_dataset_id, streaming=True, filters=filters)
+ ```
+
+ Increase the minimum request size when streaming from 32MiB (default) to 128MiB and enable prefetching:
+
+ ```python
+ >>> import pyarrow
+ >>> import pyarrow.dataset
+ >>> fragment_scan_options = pyarrow.dataset.ParquetFragmentScanOptions(
+ ... cache_options=pyarrow.CacheOptions(
+ ... prefetch_limit=1,
+ ... range_size_limit=128 << 20
+ ... ),
+ ... )
+ >>> ds = load_dataset(parquet_dataset_id, streaming=True, fragment_scan_options=fragment_scan_options)
+ ```
+
+ """
+
+ batch_size: Optional[int] = None
+ columns: Optional[list[str]] = None
+ features: Optional[datasets.Features] = None
+ filters: Optional[Union[ds.Expression, list[tuple], list[list[tuple]]]] = None
+ fragment_scan_options: Optional[ds.ParquetFragmentScanOptions] = None
+ on_bad_files: Literal["error", "warn", "skip"] = "error"
+
+ def __post_init__(self):
+ super().__post_init__()
+
+
+class Parquet(datasets.ArrowBasedBuilder):
+ BUILDER_CONFIG_CLASS = ParquetConfig
+ SLEEP_ON_THREADS_SHUTDOWNS = True # Related to https://github.com/apache/arrow/issues/45214
+
+ def _info(self):
+ if (
+ self.config.columns is not None
+ and self.config.features is not None
+ and set(self.config.columns) != set(self.config.features)
+ ):
+ if any(col not in self.config.features for col in self.config.columns):
+ raise ValueError(
+ "The columns and features argument must match, but got ",
+ f"{self.config.columns} and {self.config.features}",
+ )
+ else:
+ features = datasets.Features({col: self.config.features[col] for col in self.config.columns})
+ else:
+ features = self.config.features
+ return datasets.DatasetInfo(features=features)
+
+ def _split_generators(self, dl_manager):
+ """We handle string, list and dicts in datafiles"""
+ if not self.config.data_files:
+ raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+ dl_manager.download_config.extract_on_the_fly = True
+ data_files = dl_manager.download(self.config.data_files)
+ splits = []
+ for split_name, files in data_files.items():
+ # Infer features if they are stored in the arrow schema
+ if self.info.features is None:
+ for file in files:
+ try:
+ with open(file, "rb") as f:
+ self.info.features = datasets.Features.from_arrow_schema(pq.read_schema(f))
+ break
+ except pa.ArrowInvalid as e:
+ if self.config.on_bad_files == "error":
+ logger.error(f"Failed to read schema from '{file}' with error {type(e).__name__}: {e}")
+ raise
+ elif self.config.on_bad_files == "warn":
+ logger.warning(f"Skipping bad schema from '{file}'. {type(e).__name__}: {e}`")
+ else:
+ logger.debug(f"Skipping bad schema from '{file}'. {type(e).__name__}: {e}`")
+ if self.info.features is None:
+ raise ValueError(
+ f"At least one valid data file must be specified, all the data_files are invalid: {self.config.data_files}"
+ )
+ splits.append(
+ datasets.SplitGenerator(
+ name=split_name, gen_kwargs={"files": files, "row_groups_list": [None] * len(files)}
+ )
+ )
+ if self.config.columns is not None and set(self.config.columns) != set(self.info.features):
+ self.info.features = datasets.Features(
+ {col: feat for col, feat in self.info.features.items() if col in self.config.columns}
+ )
+ return splits
+
+ def _cast_table(self, pa_table: pa.Table) -> pa.Table:
+ if self.info.features is not None:
+ # more expensive cast to support nested features with keys in a different order
+ # allows str <-> int/float or str to Audio for example
+ pa_table = table_cast(pa_table, self.info.features.arrow_schema)
+ return pa_table
+
+ def _generate_shards(self, files, row_groups_list):
+ if not row_groups_list:
+ yield from files
+ else:
+ for file, row_groups in zip(files, row_groups_list):
+ yield {
+ "fragment_data_file": file,
+ "fragment_row_groups": row_groups,
+ }
+
+ def _generate_more_gen_kwargs(self, files, row_groups_list):
+ if not row_groups_list or any(row_group is None for row_group in row_groups_list):
+ parquet_file_format = ds.ParquetFileFormat(default_fragment_scan_options=self.config.fragment_scan_options)
+ for file in files:
+ with open(file, "rb") as f:
+ parquet_fragment = parquet_file_format.make_fragment(f)
+ yield {
+ "files": [file] * parquet_fragment.num_row_groups,
+ "row_groups_list": [
+ (row_group_id,) for row_group_id in range(parquet_fragment.num_row_groups)
+ ],
+ }
+ else:
+ for file, row_groups in zip(files, row_groups_list):
+ yield {"files": [file], "row_groups_list": [row_groups]}
+
+ def _generate_tables(self, files, row_groups_list):
+ if self.config.features is not None and self.config.columns is not None:
+ if sorted(field.name for field in self.info.features.arrow_schema) != sorted(self.config.columns):
+ raise ValueError(
+ f"Tried to load parquet data with columns '{self.config.columns}' with mismatching features '{self.info.features}'"
+ )
+ filter_expr = (
+ pq.filters_to_expression(self.config.filters)
+ if isinstance(self.config.filters, list)
+ else self.config.filters
+ )
+ parquet_file_format = ds.ParquetFileFormat(default_fragment_scan_options=self.config.fragment_scan_options)
+ for file_idx, (file, row_groups) in enumerate(zip(files, row_groups_list)):
+ try:
+ with open(file, "rb") as f:
+ parquet_fragment = parquet_file_format.make_fragment(f)
+ fragment_is_closed = False
+ try:
+ if row_groups is not None:
+ parquet_fragment = parquet_fragment.subset(row_group_ids=row_groups)
+ if parquet_fragment.row_groups:
+ batch_size = self.config.batch_size or parquet_fragment.row_groups[0].num_rows
+ for batch_idx, record_batch in enumerate(
+ parquet_fragment.to_batches(
+ batch_size=batch_size,
+ columns=self.config.columns,
+ filter=filter_expr,
+ batch_readahead=0,
+ fragment_readahead=0,
+ )
+ ):
+ pa_table = pa.Table.from_batches([record_batch])
+ # Uncomment for debugging (will print the Arrow table size and elements)
+ # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
+ # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
+ yield Key(file_idx, batch_idx), self._cast_table(pa_table)
+ fragment_is_closed = True
+ finally:
+ # Fix for https://github.com/apache/arrow/issues/45214
+ if not fragment_is_closed and datasets.config.PYARROW_VERSION <= version.parse("24.0.0"):
+ del parquet_fragment
+ gc.collect()
+ except (pa.ArrowInvalid, ValueError) as e:
+ if self.config.on_bad_files == "error":
+ logger.error(f"Failed to read file '{file}' with error {type(e).__name__}: {e}")
+ raise
+ elif self.config.on_bad_files == "warn":
+ logger.warning(f"Skipping bad file '{file}'. {type(e).__name__}: {e}`")
+ else:
+ logger.debug(f"Skipping bad file '{file}'. {type(e).__name__}: {e}`")
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/pdffolder/__init__.py b/venv/Lib/site-packages/datasets/packaged_modules/pdffolder/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/pdffolder/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/pdffolder/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..66534882fa327eb9aa5cf110b5cae996b8f23695
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/pdffolder/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/pdffolder/__pycache__/pdffolder.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/pdffolder/__pycache__/pdffolder.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b09df4c2665c212e7567bbfcc1df515b954d1f30
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/pdffolder/__pycache__/pdffolder.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/pdffolder/pdffolder.py b/venv/Lib/site-packages/datasets/packaged_modules/pdffolder/pdffolder.py
new file mode 100644
index 0000000000000000000000000000000000000000..5aaf9c71a4df6551a53a2625b332bc51a2f02837
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/pdffolder/pdffolder.py
@@ -0,0 +1,23 @@
+import datasets
+
+from ..folder_based_builder import folder_based_builder
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+class PdfFolderConfig(folder_based_builder.FolderBasedBuilderConfig):
+ """BuilderConfig for ImageFolder."""
+
+ drop_labels: bool = None
+ drop_metadata: bool = None
+
+ def __post_init__(self):
+ super().__post_init__()
+
+
+class PdfFolder(folder_based_builder.FolderBasedBuilder):
+ BASE_FEATURE = datasets.Pdf
+ BASE_COLUMN_NAME = "pdf"
+ BUILDER_CONFIG_CLASS = PdfFolderConfig
+ EXTENSIONS: list[str] = [".pdf"]
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/spark/__init__.py b/venv/Lib/site-packages/datasets/packaged_modules/spark/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/spark/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/spark/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d9db2f447298c5eeaaa1dbccd19c8e3fadf48ff8
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/spark/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/spark/__pycache__/spark.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/spark/__pycache__/spark.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6dedf2258ffc93ee6615dfd447ebe8903c56847a
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/spark/__pycache__/spark.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/spark/spark.py b/venv/Lib/site-packages/datasets/packaged_modules/spark/spark.py
new file mode 100644
index 0000000000000000000000000000000000000000..47ba9cf0dc66bf1bda3f9bb11579a8d51f661398
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/spark/spark.py
@@ -0,0 +1,367 @@
+import os
+import posixpath
+import uuid
+from collections.abc import Iterable
+from dataclasses import dataclass
+from itertools import islice
+from typing import TYPE_CHECKING, Optional, Union
+
+import numpy as np
+import pyarrow as pa
+
+import datasets
+from datasets.arrow_writer import ArrowWriter, ParquetWriter
+from datasets.config import MAX_SHARD_SIZE
+from datasets.filesystems import (
+ is_remote_filesystem,
+ rename,
+)
+from datasets.iterable_dataset import _BaseExamplesIterable
+from datasets.utils import experimental
+from datasets.utils.py_utils import convert_file_size_to_int
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+if TYPE_CHECKING:
+ import pyspark
+ import pyspark.sql
+
+
+@dataclass
+class SparkConfig(datasets.BuilderConfig):
+ """BuilderConfig for Spark."""
+
+ features: Optional[datasets.Features] = None
+
+ def __post_init__(self):
+ super().__post_init__()
+
+
+def _reorder_dataframe_by_partition(df: "pyspark.sql.DataFrame", new_partition_order: list[int]):
+ df_combined = df.select("*").where(f"part_id = {new_partition_order[0]}")
+ for partition_id in new_partition_order[1:]:
+ partition_df = df.select("*").where(f"part_id = {partition_id}")
+ df_combined = df_combined.union(partition_df)
+ return df_combined
+
+
+def _generate_iterable_examples(
+ df: "pyspark.sql.DataFrame",
+ partition_order: list[int],
+ state_dict: Optional[dict] = None,
+):
+ import pyspark
+
+ df_with_partition_id = df.select("*", pyspark.sql.functions.spark_partition_id().alias("part_id"))
+ partition_idx_start = state_dict["partition_idx"] if state_dict else 0
+ partition_df = _reorder_dataframe_by_partition(df_with_partition_id, partition_order[partition_idx_start:])
+ # pipeline next partition in parallel to hide latency
+ rows = partition_df.toLocalIterator(prefetchPartitions=True)
+ curr_partition = None
+ row_id = state_dict["partition_example_idx"] if state_dict else 0
+ for row in islice(rows, row_id, None):
+ row_as_dict = row.asDict()
+ part_id = row_as_dict["part_id"]
+ row_as_dict.pop("part_id")
+ if curr_partition != part_id:
+ if state_dict and curr_partition is not None:
+ state_dict["partition_idx"] += 1
+ curr_partition = part_id
+ row_id = 0
+ if state_dict:
+ state_dict["partition_example_idx"] = row_id + 1
+ yield (part_id, row_id), row_as_dict
+ row_id += 1
+
+
+class SparkExamplesIterable(_BaseExamplesIterable):
+ def __init__(
+ self,
+ df: "pyspark.sql.DataFrame",
+ partition_order=None,
+ ):
+ super().__init__()
+ self.df = df
+ self.partition_order = partition_order or range(self.df.rdd.getNumPartitions())
+
+ def _init_state_dict(self) -> dict:
+ self._state_dict = {"partition_idx": 0, "partition_example_idx": 0}
+ return self._state_dict
+
+ @experimental
+ def load_state_dict(self, state_dict: dict) -> dict:
+ return super().load_state_dict(state_dict)
+
+ def __iter__(self):
+ yield from _generate_iterable_examples(self.df, self.partition_order, self._state_dict)
+
+ def shuffle_data_sources(self, generator: np.random.Generator) -> "SparkExamplesIterable":
+ partition_order = list(range(self.df.rdd.getNumPartitions()))
+ generator.shuffle(partition_order)
+ return SparkExamplesIterable(self.df, partition_order=partition_order)
+
+ def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "SparkExamplesIterable":
+ partition_order = self.split_shard_indices_by_worker(num_shards=num_shards, index=index, contiguous=contiguous)
+ return SparkExamplesIterable(self.df, partition_order=partition_order)
+
+ @property
+ def num_shards(self) -> int:
+ return len(self.partition_order)
+
+
+class Spark(datasets.DatasetBuilder):
+ BUILDER_CONFIG_CLASS = SparkConfig
+
+ def __init__(
+ self,
+ df: "pyspark.sql.DataFrame",
+ cache_dir: str = None,
+ working_dir: str = None,
+ **config_kwargs,
+ ):
+ import pyspark
+
+ self._spark = pyspark.sql.SparkSession.builder.getOrCreate()
+ self.df = df
+ self._working_dir = working_dir
+
+ super().__init__(
+ cache_dir=cache_dir,
+ config_name=str(self.df.semanticHash()),
+ **config_kwargs,
+ )
+
+ def _validate_cache_dir(self):
+ # Define this so that we don't reference self in create_cache_and_write_probe, which will result in a pickling
+ # error due to pickling the SparkContext.
+ cache_dir = self._cache_dir
+
+ # Returns the path of the created file.
+ def create_cache_and_write_probe(context):
+ # makedirs with exist_ok will recursively create the directory. It will not throw an error if directories
+ # already exist.
+ os.makedirs(cache_dir, exist_ok=True)
+ probe_file = os.path.join(cache_dir, "fs_test" + uuid.uuid4().hex)
+ # Opening the file in append mode will create a new file unless it already exists, in which case it will not
+ # change the file contents.
+ open(probe_file, "a")
+ return [probe_file]
+
+ if self._spark.conf.get("spark.master", "").startswith("local"):
+ return
+
+ # If the cluster is multi-node, make sure that the user provided a cache_dir and that it is on an NFS
+ # accessible to the driver.
+ # TODO: Stream batches to the driver using ArrowCollectSerializer instead of throwing an error.
+ if self._cache_dir:
+ probe = (
+ self._spark.sparkContext.parallelize(range(1), 1).mapPartitions(create_cache_and_write_probe).collect()
+ )
+ if os.path.isfile(probe[0]):
+ return
+
+ raise ValueError(
+ "When using Dataset.from_spark on a multi-node cluster, the driver and all workers should be able to access cache_dir"
+ )
+
+ def _info(self):
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager: datasets.download.download_manager.DownloadManager):
+ return [datasets.SplitGenerator(name=datasets.Split.TRAIN)]
+
+ def _repartition_df_if_needed(self, max_shard_size):
+ import pyspark
+
+ def get_arrow_batch_size(it):
+ for batch in it:
+ yield pa.RecordBatch.from_pydict({"batch_bytes": [batch.nbytes]})
+
+ df_num_rows = self.df.count()
+ sample_num_rows = df_num_rows if df_num_rows <= 100 else 100
+ # Approximate the size of each row (in Arrow format) by averaging over a max-100-row sample.
+ approx_bytes_per_row = (
+ self.df.limit(sample_num_rows)
+ .repartition(1)
+ .mapInArrow(get_arrow_batch_size, "batch_bytes: long")
+ .agg(pyspark.sql.functions.sum("batch_bytes").alias("sample_bytes"))
+ .collect()[0]
+ .sample_bytes
+ / sample_num_rows
+ )
+ approx_total_size = approx_bytes_per_row * df_num_rows
+ if approx_total_size > max_shard_size:
+ # Make sure there is at least one row per partition.
+ new_num_partitions = min(df_num_rows, int(approx_total_size / max_shard_size))
+ self.df = self.df.repartition(new_num_partitions)
+
+ def _prepare_split_single(
+ self,
+ fpath: str,
+ file_format: str,
+ max_shard_size: int,
+ ) -> Iterable[tuple[int, bool, Union[int, tuple]]]:
+ import pyspark
+
+ writer_class = ParquetWriter if file_format == "parquet" else ArrowWriter
+ working_fpath = os.path.join(self._working_dir, os.path.basename(fpath)) if self._working_dir else fpath
+ embed_local_files = file_format == "parquet"
+
+ # Define these so that we don't reference self in write_arrow, which will result in a pickling error due to
+ # pickling the SparkContext.
+ features = self.config.features
+ writer_batch_size = self._writer_batch_size
+ storage_options = self._fs.storage_options
+
+ def write_arrow(it):
+ # Within the same SparkContext, no two task attempts will share the same attempt ID.
+ task_id = pyspark.TaskContext().taskAttemptId()
+ first_batch = next(it, None)
+ if first_batch is None:
+ # Some partitions might not receive any data.
+ return pa.RecordBatch.from_arrays(
+ [[task_id], [0], [0]],
+ names=["task_id", "num_examples", "num_bytes"],
+ )
+ shard_id = 0
+ writer = writer_class(
+ features=features,
+ path=working_fpath.replace("SSSSS", f"{shard_id:05d}").replace("TTTTT", f"{task_id:05d}"),
+ writer_batch_size=writer_batch_size,
+ storage_options=storage_options,
+ embed_local_files=embed_local_files,
+ )
+ table = pa.Table.from_batches([first_batch])
+ writer.write_table(table)
+ for batch in it:
+ if max_shard_size is not None and writer._num_bytes >= max_shard_size:
+ num_examples, num_bytes = writer.finalize()
+ writer.close()
+ yield pa.RecordBatch.from_arrays(
+ [[task_id], [num_examples], [num_bytes]],
+ names=["task_id", "num_examples", "num_bytes"],
+ )
+ shard_id += 1
+ writer = writer_class(
+ features=writer._features,
+ path=working_fpath.replace("SSSSS", f"{shard_id:05d}").replace("TTTTT", f"{task_id:05d}"),
+ writer_batch_size=writer_batch_size,
+ storage_options=storage_options,
+ embed_local_files=embed_local_files,
+ )
+ table = pa.Table.from_batches([batch])
+ writer.write_table(table)
+
+ if writer._num_bytes > 0:
+ num_examples, num_bytes = writer.finalize()
+ writer.close()
+ yield pa.RecordBatch.from_arrays(
+ [[task_id], [num_examples], [num_bytes]],
+ names=["task_id", "num_examples", "num_bytes"],
+ )
+
+ if working_fpath != fpath:
+ for file in os.listdir(os.path.dirname(working_fpath)):
+ dest = os.path.join(os.path.dirname(fpath), os.path.basename(file))
+ shutil.move(file, dest)
+
+ stats = (
+ self.df.mapInArrow(write_arrow, "task_id: long, num_examples: long, num_bytes: long")
+ .groupBy("task_id")
+ .agg(
+ pyspark.sql.functions.sum("num_examples").alias("total_num_examples"),
+ pyspark.sql.functions.sum("num_bytes").alias("total_num_bytes"),
+ pyspark.sql.functions.count("num_bytes").alias("num_shards"),
+ pyspark.sql.functions.collect_list("num_examples").alias("shard_lengths"),
+ )
+ .collect()
+ )
+ for row in stats:
+ yield row.task_id, (row.total_num_examples, row.total_num_bytes, row.num_shards, row.shard_lengths)
+
+ def _prepare_split(
+ self,
+ split_generator: "datasets.SplitGenerator",
+ file_format: str = "arrow",
+ max_shard_size: Optional[Union[str, int]] = None,
+ num_proc: Optional[int] = None,
+ **kwargs,
+ ):
+ self._validate_cache_dir()
+
+ max_shard_size = convert_file_size_to_int(max_shard_size or MAX_SHARD_SIZE)
+ self._repartition_df_if_needed(max_shard_size)
+ is_local = not is_remote_filesystem(self._fs)
+ path_join = os.path.join if is_local else posixpath.join
+
+ SUFFIX = "-TTTTT-SSSSS-of-NNNNN"
+ fname = f"{self.name}-{split_generator.name}{SUFFIX}.{file_format}"
+ fpath = path_join(self._output_dir, fname)
+
+ total_num_examples = 0
+ total_num_bytes = 0
+ total_shards = 0
+ task_id_and_num_shards = []
+ all_shard_lengths = []
+
+ for task_id, content in self._prepare_split_single(fpath, file_format, max_shard_size):
+ (
+ num_examples,
+ num_bytes,
+ num_shards,
+ shard_lengths,
+ ) = content
+ if num_bytes > 0:
+ total_num_examples += num_examples
+ total_num_bytes += num_bytes
+ total_shards += num_shards
+ task_id_and_num_shards.append((task_id, num_shards))
+ all_shard_lengths.extend(shard_lengths)
+
+ split_generator.split_info.num_examples = total_num_examples
+ split_generator.split_info.num_bytes = total_num_bytes
+
+ # should rename everything at the end
+ logger.debug(f"Renaming {total_shards} shards.")
+ if total_shards > 1:
+ split_generator.split_info.shard_lengths = all_shard_lengths
+
+ # Define fs outside of _rename_shard so that we don't reference self in the function, which will result in a
+ # pickling error due to pickling the SparkContext.
+ fs = self._fs
+
+ # use the -SSSSS-of-NNNNN pattern
+ def _rename_shard(
+ task_id: int,
+ shard_id: int,
+ global_shard_id: int,
+ ):
+ rename(
+ fs,
+ fpath.replace("SSSSS", f"{shard_id:05d}").replace("TTTTT", f"{task_id:05d}"),
+ fpath.replace("TTTTT-SSSSS", f"{global_shard_id:05d}").replace("NNNNN", f"{total_shards:05d}"),
+ )
+
+ args = []
+ global_shard_id = 0
+ for i in range(len(task_id_and_num_shards)):
+ task_id, num_shards = task_id_and_num_shards[i]
+ for shard_id in range(num_shards):
+ args.append([task_id, shard_id, global_shard_id])
+ global_shard_id += 1
+ self._spark.sparkContext.parallelize(args, len(args)).map(lambda args: _rename_shard(*args)).collect()
+ else:
+ # don't use any pattern
+ shard_id = 0
+ task_id = task_id_and_num_shards[0][0]
+ self._rename(
+ fpath.replace("SSSSS", f"{shard_id:05d}").replace("TTTTT", f"{task_id:05d}"),
+ fpath.replace(SUFFIX, ""),
+ )
+
+ def _get_examples_iterable_for_split(
+ self,
+ split_generator: "datasets.SplitGenerator",
+ ) -> SparkExamplesIterable:
+ return SparkExamplesIterable(self.df)
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/sql/__init__.py b/venv/Lib/site-packages/datasets/packaged_modules/sql/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/sql/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/sql/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..a8f29ef985b0b4de7e1e97925da035e0ca76312a
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/sql/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/sql/__pycache__/sql.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/sql/__pycache__/sql.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d78a22fd6c1110f725e20ac549333f3ac699c2a1
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/sql/__pycache__/sql.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/sql/sql.py b/venv/Lib/site-packages/datasets/packaged_modules/sql/sql.py
new file mode 100644
index 0000000000000000000000000000000000000000..deebd41428d8f89d74a75e67729be1dc1ed1a468
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/sql/sql.py
@@ -0,0 +1,120 @@
+import sys
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Optional, Union
+
+import pandas as pd
+import pyarrow as pa
+
+import datasets
+import datasets.config
+from datasets.builder import Key
+from datasets.features.features import require_storage_cast
+from datasets.table import table_cast
+
+
+if TYPE_CHECKING:
+ import sqlite3
+
+ import sqlalchemy
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+@dataclass
+class SqlConfig(datasets.BuilderConfig):
+ """BuilderConfig for SQL."""
+
+ sql: Union[str, "sqlalchemy.sql.Selectable"] = None
+ con: Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"] = None
+ index_col: Optional[Union[str, list[str]]] = None
+ coerce_float: bool = True
+ params: Optional[Union[list, tuple, dict]] = None
+ parse_dates: Optional[Union[list, dict]] = None
+ columns: Optional[list[str]] = None
+ chunksize: Optional[int] = 10_000
+ features: Optional[datasets.Features] = None
+
+ def __post_init__(self):
+ super().__post_init__()
+ if self.sql is None:
+ raise ValueError("sql must be specified")
+ if self.con is None:
+ raise ValueError("con must be specified")
+
+ def create_config_id(
+ self,
+ config_kwargs: dict,
+ custom_features: Optional[datasets.Features] = None,
+ ) -> str:
+ config_kwargs = config_kwargs.copy()
+ # We need to stringify the Selectable object to make its hash deterministic
+
+ # The process of stringifying is explained here: http://docs.sqlalchemy.org/en/latest/faq/sqlexpressions.html
+ sql = config_kwargs["sql"]
+ if not isinstance(sql, str):
+ if datasets.config.SQLALCHEMY_AVAILABLE and "sqlalchemy" in sys.modules:
+ import sqlalchemy
+
+ if isinstance(sql, sqlalchemy.sql.Selectable):
+ engine = sqlalchemy.create_engine(config_kwargs["con"].split("://")[0] + "://")
+ sql_str = str(sql.compile(dialect=engine.dialect))
+ config_kwargs["sql"] = sql_str
+ else:
+ raise TypeError(
+ f"Supported types for 'sql' are string and sqlalchemy.sql.Selectable but got {type(sql)}: {sql}"
+ )
+ else:
+ raise TypeError(
+ f"Supported types for 'sql' are string and sqlalchemy.sql.Selectable but got {type(sql)}: {sql}"
+ )
+ con = config_kwargs["con"]
+ if not isinstance(con, str):
+ config_kwargs["con"] = id(con)
+ logger.info(
+ f"SQL connection 'con' of type {type(con)} couldn't be hashed properly. To enable hashing, specify 'con' as URI string instead."
+ )
+
+ return super().create_config_id(config_kwargs, custom_features=custom_features)
+
+ @property
+ def pd_read_sql_kwargs(self):
+ pd_read_sql_kwargs = {
+ "index_col": self.index_col,
+ "columns": self.columns,
+ "params": self.params,
+ "coerce_float": self.coerce_float,
+ "parse_dates": self.parse_dates,
+ }
+ return pd_read_sql_kwargs
+
+
+class Sql(datasets.ArrowBasedBuilder):
+ BUILDER_CONFIG_CLASS = SqlConfig
+
+ def _info(self):
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager):
+ return [datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={})]
+
+ def _cast_table(self, pa_table: pa.Table) -> pa.Table:
+ if self.config.features is not None:
+ schema = self.config.features.arrow_schema
+ if all(not require_storage_cast(feature) for feature in self.config.features.values()):
+ # cheaper cast
+ pa_table = pa.Table.from_arrays([pa_table[field.name] for field in schema], schema=schema)
+ else:
+ # more expensive cast; allows str <-> int/float or str to Audio for example
+ pa_table = table_cast(pa_table, schema)
+ return pa_table
+
+ def _generate_tables(self):
+ chunksize = self.config.chunksize
+ sql_reader = pd.read_sql(
+ self.config.sql, self.config.con, chunksize=chunksize, **self.config.pd_read_sql_kwargs
+ )
+ sql_reader = [sql_reader] if chunksize is None else sql_reader
+ for chunk_idx, df in enumerate(sql_reader):
+ pa_table = pa.Table.from_pandas(df)
+ yield Key(0, chunk_idx), self._cast_table(pa_table)
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/text/__init__.py b/venv/Lib/site-packages/datasets/packaged_modules/text/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/text/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/text/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9e10c153a00c2bbd355d7f4bc1cecbcfc5aa9c14
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/text/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/text/__pycache__/text.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/text/__pycache__/text.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0b2841ecfffaaff502737ad3979b58bd08602363
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/text/__pycache__/text.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/text/text.py b/venv/Lib/site-packages/datasets/packaged_modules/text/text.py
new file mode 100644
index 0000000000000000000000000000000000000000..5c6ac72ca3cc8b4e9c4b60019ce8dd04c0444a56
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/text/text.py
@@ -0,0 +1,137 @@
+from dataclasses import dataclass
+from io import StringIO
+from typing import Literal, Optional
+
+import pyarrow as pa
+
+import datasets
+from datasets.builder import Key
+from datasets.features.features import require_storage_cast
+from datasets.table import table_cast
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+@dataclass
+class TextConfig(datasets.BuilderConfig):
+ """BuilderConfig for text files.
+
+ Args:
+ features: (`Features`, *optional*):
+ Cast the data to `features`.
+ encoding: (`str`, defaults to "utf-8"):
+ Encoding to decode the file.
+ encoding_errors: (`str`, *optional*):
+ Argument to define what to do in case of encoding error.
+ This is the same as the `error` argument in `open()`.
+ chunksize: (`Features`, *optional*, defaults to "10MB"):
+ Chunk size to read the data.
+ keep_linebreaks: (`bool`, defaults to False):
+ Whether to keep line breaks.
+ sample_by (`Literal["line", "paragraph", "document"]`, defaults to "line"):
+ Whether to load data per line, praragraph or document.
+ By default one row in the dataset = one line.
+ """
+
+ features: Optional[datasets.Features] = None
+ encoding: str = "utf-8"
+ encoding_errors: Optional[str] = None
+ chunksize: int = 10 << 20 # 10MB
+ keep_linebreaks: bool = False
+ sample_by: Literal["line", "paragraph", "document"] = "line"
+
+
+class Text(datasets.ArrowBasedBuilder):
+ BUILDER_CONFIG_CLASS = TextConfig
+
+ def _info(self):
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager):
+ """The `data_files` kwarg in load_dataset() can be a str, List[str], Dict[str,str], or Dict[str,List[str]].
+
+ If str or List[str], then the dataset returns only the 'train' split.
+ If dict, then keys should be from the `datasets.Split` enum.
+ """
+ if not self.config.data_files:
+ raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+ dl_manager.download_config.extract_on_the_fly = True
+ base_data_files = dl_manager.download(self.config.data_files)
+ extracted_data_files = dl_manager.extract(base_data_files)
+ splits = []
+ for split_name, files in extracted_data_files.items():
+ files_iterables = [dl_manager.iter_files(file) for file in files]
+ splits.append(
+ datasets.SplitGenerator(
+ name=split_name,
+ gen_kwargs={"files_iterables": files_iterables, "base_files": base_data_files[split_name]},
+ )
+ )
+ return splits
+
+ def _cast_table(self, pa_table: pa.Table) -> pa.Table:
+ if self.config.features is not None:
+ schema = self.config.features.arrow_schema
+ if all(not require_storage_cast(feature) for feature in self.config.features.values()):
+ # cheaper cast
+ pa_table = pa_table.cast(schema)
+ else:
+ # more expensive cast; allows str <-> int/float or str to Audio for example
+ pa_table = table_cast(pa_table, schema)
+ return pa_table
+ else:
+ return pa_table.cast(pa.schema({"text": pa.string()}))
+
+ def _generate_shards(self, base_files, files_iterables):
+ yield from base_files
+
+ def _generate_tables(self, base_files, files_iterables):
+ pa_table_names = list(self.config.features) if self.config.features is not None else ["text"]
+ for shard_idx, files_iterable in enumerate(files_iterables):
+ for file in files_iterable:
+ # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
+ with open(file, encoding=self.config.encoding, errors=self.config.encoding_errors) as f:
+ if self.config.sample_by == "line":
+ batch_idx = 0
+ while True:
+ batch = f.read(self.config.chunksize)
+ if not batch:
+ break
+ batch += f.readline() # finish current line
+ # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
+ batch = StringIO(batch).readlines()
+ if not self.config.keep_linebreaks:
+ batch = [line.rstrip("\n") for line in batch]
+ pa_table = pa.Table.from_arrays([pa.array(batch)], names=pa_table_names)
+ # Uncomment for debugging (will print the Arrow table size and elements)
+ # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
+ # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
+ yield Key(shard_idx, batch_idx), self._cast_table(pa_table)
+ batch_idx += 1
+ elif self.config.sample_by == "paragraph":
+ batch_idx = 0
+ batch = ""
+ while True:
+ new_batch = f.read(self.config.chunksize)
+ if not new_batch:
+ break
+ batch += new_batch
+ batch += f.readline() # finish current line
+ batch = batch.split("\n\n")
+ pa_table = pa.Table.from_arrays(
+ [pa.array([example for example in batch[:-1] if example])], names=pa_table_names
+ )
+ # Uncomment for debugging (will print the Arrow table size and elements)
+ # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
+ # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
+ yield Key(shard_idx, batch_idx), self._cast_table(pa_table)
+ batch_idx += 1
+ batch = batch[-1]
+ if batch:
+ pa_table = pa.Table.from_arrays([pa.array([batch])], names=pa_table_names)
+ yield (shard_idx, batch_idx), self._cast_table(pa_table)
+ elif self.config.sample_by == "document":
+ text = f.read()
+ pa_table = pa.Table.from_arrays([pa.array([text])], names=pa_table_names)
+ yield Key(shard_idx, 0), self._cast_table(pa_table)
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/tsfile/__init__.py b/venv/Lib/site-packages/datasets/packaged_modules/tsfile/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/tsfile/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/tsfile/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f3fdcbd879d546215ee8361c0da99987d840cc1e
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/tsfile/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/tsfile/__pycache__/tsfile.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/tsfile/__pycache__/tsfile.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4c802f05268510633d3abd8cdec9534c5a5d8847
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/tsfile/__pycache__/tsfile.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/tsfile/tsfile.py b/venv/Lib/site-packages/datasets/packaged_modules/tsfile/tsfile.py
new file mode 100644
index 0000000000000000000000000000000000000000..96712b4516e111bb186e83c764ccec3062dba9b8
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/tsfile/tsfile.py
@@ -0,0 +1,773 @@
+"""TsFile (table model) packaged builder — per-device wide format.
+
+Each output row corresponds to a single device (identified by its TAG values).
+The ``time`` column and every FIELD column are Arrow ``list<...>`` columns
+holding the entire time series for that device. When the same device is
+present in multiple TsFiles within a split, its data is merged across files
+and the resulting lists are sorted in ascending time order.
+
+Output schema layout::
+
+ : string
+ : string (one column per TAG)
+ ...
+ time: list
+ : list (one column per FIELD)
+ : list
+ ...
+
+Reading model
+-------------
+Data is fetched **per device** via ``TsFileReader.query_table`` with a
+push-down ``tag_filter``. For each split the builder:
+
+1. Opens every input file once, calls ``get_all_devices`` to enumerate the
+ ``(tag-tuple) → [files]`` index across all shards.
+2. Iterates the index in stable order. For each device, streams Arrow
+ batches from every contributing file, concatenates and sorts by time,
+ and emits one wide row.
+
+Peak memory is bounded by **one device's** total payload across the split,
+not by the split's total size.
+"""
+
+from __future__ import annotations
+
+import datetime as _dt
+from dataclasses import dataclass
+from typing import Any, Literal, Optional
+
+import numpy as np
+import pyarrow as pa
+
+import datasets
+from datasets.builder import Key
+from datasets.table import table_cast
+from datasets.utils.tqdm import tqdm
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Type helpers
+# ---------------------------------------------------------------------------
+
+
+def _arrow_type(ts_dtype, *, unit: str, tz: Optional[str]) -> pa.DataType:
+ """Map a tsfile ``TSDataType`` to its Arrow representation."""
+ from tsfile.constants import TSDataType
+
+ return {
+ TSDataType.BOOLEAN: pa.bool_(),
+ TSDataType.INT32: pa.int32(),
+ TSDataType.INT64: pa.int64(),
+ TSDataType.FLOAT: pa.float32(),
+ TSDataType.DOUBLE: pa.float64(),
+ TSDataType.TEXT: pa.string(),
+ TSDataType.STRING: pa.string(),
+ TSDataType.TIMESTAMP: pa.timestamp(unit, tz=tz),
+ TSDataType.DATE: pa.date32(),
+ TSDataType.BLOB: pa.binary(),
+ }.get(ts_dtype, pa.string())
+
+
+def _promote_tsdatatype(a, b):
+ """Return the widest of two ``TSDataType`` values.
+
+ Mirrors IoTDB's ``ALTER COLUMN ... SET DATA TYPE`` rules:
+
+ - ``INT32 → INT64 → DOUBLE``
+ - ``INT32 → FLOAT → DOUBLE``
+
+ ``INT64`` and ``FLOAT`` cannot widen losslessly into either, so their
+ join is ``DOUBLE``. Non-numeric or otherwise unrelated pairs raise.
+ """
+ if a == b:
+ return a
+
+ from tsfile.constants import TSDataType
+
+ table = {
+ (TSDataType.INT32, TSDataType.INT64): TSDataType.INT64,
+ (TSDataType.INT32, TSDataType.FLOAT): TSDataType.FLOAT,
+ (TSDataType.INT32, TSDataType.DOUBLE): TSDataType.DOUBLE,
+ (TSDataType.INT64, TSDataType.FLOAT): TSDataType.DOUBLE,
+ (TSDataType.INT64, TSDataType.DOUBLE): TSDataType.DOUBLE,
+ (TSDataType.FLOAT, TSDataType.DOUBLE): TSDataType.DOUBLE,
+ }
+ if (a, b) in table:
+ return table[(a, b)]
+ if (b, a) in table:
+ return table[(b, a)]
+ raise ValueError(
+ f"Incompatible column types across files: {a.name} vs {b.name}. "
+ "Only numeric widening (INT32→INT64→DOUBLE, INT32→FLOAT→DOUBLE) is supported."
+ )
+
+
+def _to_epoch(value: Any, unit: str) -> int:
+ """Coerce a timestamp boundary to an integer epoch in ``unit``.
+
+ Accepts ``int`` (raw epoch in ``unit``), ``datetime``/``date``,
+ ISO-8601 ``str``, or any ``pa.Scalar`` of timestamp type.
+ """
+ if isinstance(value, bool): # bool is a subclass of int; reject explicitly
+ raise TypeError(f"start_time/end_time must be a timestamp, got bool: {value!r}")
+ if isinstance(value, int):
+ return value
+ try:
+ # Normalize the various input shapes into something pa.scalar() can absorb
+ # under a `timestamp[unit]` target type.
+ if isinstance(value, _dt.datetime):
+ if value.tzinfo is not None:
+ value = value.astimezone(_dt.timezone.utc).replace(tzinfo=None)
+ elif isinstance(value, _dt.date):
+ value = _dt.datetime(value.year, value.month, value.day)
+ elif isinstance(value, str):
+ value = _dt.datetime.fromisoformat(value)
+ return pa.scalar(value, type=pa.timestamp(unit)).value
+ except (pa.ArrowInvalid, pa.ArrowTypeError, TypeError, ValueError) as e:
+ raise TypeError(
+ f"start_time/end_time must be a datetime, date, pa.TimestampScalar, "
+ f"ISO-8601 str, or int epoch; got {type(value).__name__}: {value!r}"
+ ) from e
+
+
+# ---------------------------------------------------------------------------
+# Config
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class TsFileConfig(datasets.BuilderConfig):
+ """BuilderConfig for TsFile (table model) — per-device wide format.
+
+ Args:
+ table_name (`str`, *optional*):
+ Name of the table to read. When unset, the first table found in
+ the first valid file is used. Lookups are case-insensitive.
+ columns (`list[str]`, *optional*):
+ Subset of FIELD columns to keep. TAG columns and the TIME column
+ are *always* returned (they identify the device / its timeline
+ and cannot be excluded). Names that refer to TAG or TIME columns,
+ or to fields absent from every file, resolve quietly: TAGs/TIME
+ are emitted as usual, and never-seen fields become all-null list
+ columns. When unset, all FIELDs are returned.
+ start_time, end_time (`datetime`, `date`, `pa.TimestampScalar`, ISO-8601 `str`, or `int`, *optional*):
+ Inclusive timestamp range. Either bound may be omitted.
+ ``datetime`` values are taken in their own tz (UTC if naive);
+ ``int`` is interpreted as a raw epoch in ``timestamp_unit``.
+ input_batch_size (`int`, *optional*, defaults to 65_536):
+ Maximum number of rows fetched per Arrow batch from
+ ``TsFileReader.query_table``. Controls peak memory while
+ streaming a single device.
+ output_batch_size (`int`, *optional*, defaults to 32):
+ Number of devices (output dataset rows) packed into each Arrow
+ record batch yielded to the writer. Also the granularity at
+ which the dataset progress bar advances; smaller values give
+ more responsive feedback on slow per-device reads, larger ones
+ reduce per-batch overhead.
+ features (`Features`, *optional*):
+ Final Features schema. When provided, the metadata scan over
+ input files is skipped.
+ on_bad_files (`Literal["error", "warn", "skip"]`, *optional*, defaults to "error"):
+ What to do if a file cannot be opened or lacks the requested table.
+ timestamp_unit (`Literal["s", "ms", "us", "ns"]`, *optional*, defaults to "ms"):
+ Time unit for the timestamp column. IoTDB defaults to milliseconds.
+ timestamp_tz (`str`, *optional*):
+ Time zone for the timestamp column. ``None`` means timezone-naive.
+ """
+
+ table_name: Optional[str] = None
+ columns: Optional[list[str]] = None
+ start_time: Optional[Any] = None
+ end_time: Optional[Any] = None
+ input_batch_size: int = 65_536
+ output_batch_size: int = 32
+ features: Optional[datasets.Features] = None
+ on_bad_files: Literal["error", "warn", "skip"] = "error"
+ timestamp_unit: Literal["s", "ms", "us", "ns"] = "ms"
+ timestamp_tz: Optional[str] = None
+
+ def __post_init__(self):
+ super().__post_init__()
+ if self.input_batch_size is None or self.input_batch_size <= 0:
+ raise ValueError(f"`input_batch_size` must be a positive integer, got {self.input_batch_size}")
+ if self.output_batch_size is None or self.output_batch_size <= 0:
+ raise ValueError(f"`output_batch_size` must be a positive integer, got {self.output_batch_size}")
+ if self.columns is not None and len(self.columns) == 0:
+ raise ValueError("`columns` must be a non-empty list when provided.")
+ if self.timestamp_unit not in ("s", "ms", "us", "ns"):
+ raise ValueError(f"`timestamp_unit` must be one of 's', 'ms', 'us', 'ns', got {self.timestamp_unit!r}")
+ if self.on_bad_files not in ("error", "warn", "skip"):
+ raise ValueError(f"`on_bad_files` must be one of 'error', 'warn', 'skip', got {self.on_bad_files!r}")
+ if self.start_time is not None:
+ self.start_time = _to_epoch(self.start_time, self.timestamp_unit)
+ if self.end_time is not None:
+ self.end_time = _to_epoch(self.end_time, self.timestamp_unit)
+
+
+# ---------------------------------------------------------------------------
+# Internal sentinels
+# ---------------------------------------------------------------------------
+
+
+class _SkipSplit(Exception):
+ """Raised internally to abort emitting a split entirely."""
+
+
+class _MissingTableError(ValueError):
+ def __init__(self, table: Optional[str], available):
+ super().__init__(f"Table {table!r} not found in file. Available tables: {available}")
+
+
+_TSFILE_MAGIC = b"TsFile"
+
+
+# ---------------------------------------------------------------------------
+# Builder
+# ---------------------------------------------------------------------------
+
+
+class TsFile(datasets.ArrowBasedBuilder):
+ """Per-device wide-format builder for TsFile (table model)."""
+
+ BUILDER_CONFIG_CLASS = TsFileConfig
+
+ # ----- builder hooks ------------------------------------------------
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._table: Optional[str] = None
+ self._time_col: str = "time"
+ self._tag_cols: list[str] = []
+ self._field_inner: dict[str, pa.DataType] = {}
+ self._requested_fields: Optional[list[str]] = None # lowercased
+
+ def _info(self):
+ if (
+ self.config.columns is not None
+ and self.config.features is not None
+ and not set(self.config.columns).issubset(set(self.config.features))
+ ):
+ raise ValueError(
+ "Every entry in `columns` must also appear in `features`, but got "
+ f"columns={self.config.columns} and features={list(self.config.features)}"
+ )
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager):
+ if not self.config.data_files:
+ raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+ dl_manager.download_config.extract_on_the_fly = True
+ data_files = dl_manager.download(self.config.data_files)
+
+ # Lowercase user-facing names to match tsfile's case-insensitive convention.
+ self._table = self.config.table_name.lower() if self.config.table_name else None
+ self._requested_fields = [c.lower() for c in self.config.columns] if self.config.columns else None
+
+ all_files = [f for files in data_files.values() for f in files]
+ scan = self._scan_metadata(all_files)
+ if scan is None:
+ raise ValueError(
+ "Could not infer schema from any of the provided files. "
+ "Set `features` explicitly or check the input files."
+ )
+ self._table = scan["table"]
+ self._time_col = scan["time_col"]
+ self._tag_cols = scan["tag_cols"]
+ self._field_inner = scan["field_inner"]
+
+ if self.info.features is None:
+ self.info.features = self._build_features()
+
+ return [
+ datasets.SplitGenerator(name=split, gen_kwargs={"files": list(files)})
+ for split, files in data_files.items()
+ ]
+
+ def _generate_shards(self, files):
+ yield from files
+
+ def _generate_tables(self, files):
+ target_schema = self.info.features.arrow_schema
+ try:
+ yield from self._fold_split(files, target_schema)
+ except _SkipSplit:
+ return
+
+ # ----- metadata scan ------------------------------------------------
+
+ def _scan_metadata(self, files) -> Optional[dict]:
+ """Walk every file and unify table name, TAG columns, FIELD types."""
+ from tsfile.constants import TIME_COLUMN, ColumnCategory
+
+ wanted_table = self._table
+ wanted_fields = set(self._requested_fields) if self._requested_fields is not None else None
+
+ table: Optional[str] = wanted_table
+ time_col: Optional[str] = None
+ tag_cols: list[str] = []
+ tag_seen: set[str] = set()
+ # Per-field widest TSDataType seen so far (we map to Arrow at the end).
+ field_widest: dict = {}
+
+ for file in files:
+ try:
+ with self._open_reader(file) as reader:
+ schemas = self._schemas_by_lc(reader)
+ self._require_table_model(file, schemas)
+ if table is None:
+ table = next(iter(schemas))
+ if table not in schemas:
+ raise _MissingTableError(table, list(schemas))
+ for col in schemas[table].get_columns():
+ name = col.get_column_name()
+ cat = col.get_category()
+ ts_dtype = col.get_data_type()
+ if cat == ColumnCategory.TIME:
+ time_col = name
+ elif cat == ColumnCategory.TAG:
+ if name not in tag_seen:
+ tag_seen.add(name)
+ tag_cols.append(name)
+ else: # FIELD
+ if wanted_fields is not None and name not in wanted_fields:
+ continue
+ prev = field_widest.get(name)
+ field_widest[name] = ts_dtype if prev is None else _promote_tsdatatype(prev, ts_dtype)
+ except Exception as e:
+ if self._should_reraise(file, e):
+ raise
+ continue
+
+ if table is None:
+ return None
+
+ unit = self.config.timestamp_unit
+ tz = self.config.timestamp_tz
+
+ if self._requested_fields is not None:
+ # Honor user order; silently drop names that turned out to be TAGs
+ # or the TIME column (TAGs are emitted as their own scalar columns
+ # and TIME is always emitted as a list column — neither may also
+ # appear as a list-typed field, which would collide on schema name).
+ reserved = tag_seen | {time_col} if time_col is not None else tag_seen
+ field_inner: dict[str, pa.DataType] = {}
+ for name in self._requested_fields:
+ if name in reserved:
+ continue
+ ts_dtype = field_widest.get(name)
+ if ts_dtype is not None:
+ field_inner[name] = _arrow_type(ts_dtype, unit=unit, tz=tz)
+ else:
+ # Field never appeared in any file — keep as a nullable
+ # float64 list, fully filled with nulls at read time.
+ field_inner[name] = pa.float64()
+ else:
+ field_inner = {n: _arrow_type(d, unit=unit, tz=tz) for n, d in field_widest.items()}
+
+ return {
+ "table": table,
+ "time_col": time_col or TIME_COLUMN,
+ "tag_cols": tag_cols,
+ "field_inner": field_inner,
+ }
+
+ def _build_features(self) -> datasets.Features:
+ unit = self.config.timestamp_unit
+ tz = self.config.timestamp_tz
+ fields: list[pa.Field] = [pa.field(t, pa.string()) for t in self._tag_cols]
+ fields.append(pa.field(self._time_col, pa.list_(pa.timestamp(unit, tz=tz))))
+ for name, inner in self._field_inner.items():
+ fields.append(pa.field(name, pa.list_(inner)))
+ return datasets.Features.from_arrow_schema(pa.schema(fields))
+
+ # ----- per-split folding -------------------------------------------
+
+ def _fold_split(self, files, target_schema: pa.Schema):
+ """Stream every device in this split via per-device tag-filter pushdown.
+
+ Open one ``TsFileReader`` per file, build a cross-file device index
+ keyed by ``(tag-tuple)``, then iterate devices in stable order. For
+ each device, ``query_table(tag_filter=...)`` reads only that device's
+ rows from each contributing file, so peak memory is bounded by one
+ device's payload across the split — never the split's total size.
+ """
+ if self._table is None:
+ raise _SkipSplit
+
+ readers: dict[str, Any] = {}
+ try:
+ for file in files:
+ try:
+ readers[file] = self._open_reader(file)
+ except Exception as e:
+ if self._should_reraise(file, e):
+ raise
+ continue
+
+ device_index, file_meta = self._build_device_index(readers)
+ if not device_index:
+ return
+
+ yield from self._iter_device_batches(device_index, file_meta, readers, target_schema)
+ finally:
+ for reader in readers.values():
+ try:
+ reader.close()
+ except Exception:
+ pass
+
+ def _build_device_index(self, readers: dict):
+ """Walk every open reader and build the cross-file device index.
+
+ Returns ``(device_index, file_meta)``:
+
+ - ``device_index``: list of ``(device_key, [file_path, ...])`` pairs
+ in stable first-seen order. ``device_key`` is a tuple aligned to
+ ``self._tag_cols`` (the unified tag-column order).
+ - ``file_meta``: maps each readable file to its per-file context
+ (``tag_cols``, ``field_cols``, ``time_col``).
+ """
+ from tsfile.constants import ColumnCategory
+
+ device_to_files: dict[tuple, list[str]] = {}
+ device_order: list[tuple] = []
+ file_meta: dict[str, dict] = {}
+ # ``self._table`` was lowercased either by user-input normalization in
+ # ``_split_generators`` or by ``_schemas_by_lc`` during auto-detect.
+ table_lc = self._table
+
+ files_iter = tqdm(
+ readers.items(),
+ total=len(readers),
+ desc="Indexing TsFile devices",
+ unit="file",
+ )
+
+ for file, reader in files_iter:
+ try:
+ schemas = self._schemas_by_lc(reader)
+ self._require_table_model(file, schemas)
+ if table_lc not in schemas:
+ raise _MissingTableError(table_lc, list(schemas))
+ schema = schemas[table_lc]
+
+ file_tag_cols: list[str] = []
+ file_field_cols: set[str] = set()
+ time_col = self._time_col
+ for col in schema.get_columns():
+ name = col.get_column_name()
+ cat = col.get_category()
+ if cat == ColumnCategory.TIME:
+ time_col = name
+ elif cat == ColumnCategory.TAG:
+ file_tag_cols.append(name)
+ elif cat == ColumnCategory.FIELD:
+ file_field_cols.add(name)
+
+ file_meta[file] = {
+ "tag_cols": file_tag_cols,
+ "field_cols": file_field_cols,
+ "time_col": time_col,
+ }
+
+ for device in reader.get_all_devices():
+ if device.table_name is None or device.table_name.lower() != table_lc:
+ continue
+ file_tag_values = list(device.segments[1 : 1 + len(file_tag_cols)])
+ file_tag_dict = dict(zip(file_tag_cols, file_tag_values))
+ unified_key = tuple(file_tag_dict.get(c) for c in self._tag_cols)
+ if any(v is None for v in unified_key):
+ raise ValueError(
+ f"Device in file '{file}' has missing tag values: "
+ f"{dict(zip(self._tag_cols, unified_key))}. "
+ "Schema-evolution devices with NULL tag values are not "
+ "supported because tsfile lacks an IS NULL tag filter."
+ )
+ if unified_key not in device_to_files:
+ device_to_files[unified_key] = []
+ device_order.append(unified_key)
+ device_to_files[unified_key].append(file)
+ except Exception as e:
+ if self._should_reraise(file, e):
+ raise
+ file_meta.pop(file, None)
+ continue
+
+ device_index = [(key, device_to_files[key]) for key in device_order]
+ return device_index, file_meta
+
+ def _iter_device_batches(self, device_index, file_meta, readers, target_schema: pa.Schema):
+ """Materialize devices in order and emit packed Arrow tables."""
+ field_names = list(self._field_inner.keys())
+ rows: list[dict] = []
+ batch_idx = 0
+ for device_key, contributing_files in device_index:
+ time_chunks: list[np.ndarray] = []
+ field_chunks: dict[str, list] = {f: [] for f in field_names}
+ for file in contributing_files:
+ if file not in file_meta:
+ continue
+ try:
+ ts_arr, vals = self._read_device_from_file(readers[file], file_meta[file], device_key)
+ except Exception as e:
+ if self._should_reraise(file, e):
+ raise
+ continue
+ if len(ts_arr) == 0:
+ continue
+ time_chunks.append(ts_arr)
+ for f in field_names:
+ field_chunks[f].append(vals.get(f)) # None → all-null contribution
+ if not time_chunks:
+ continue # device produced no rows in time range → skip
+
+ row = self._finalize_device(device_key, time_chunks, field_chunks, field_names)
+ rows.append(row)
+ if len(rows) >= self.config.output_batch_size:
+ yield Key(0, batch_idx), self._rows_to_table(rows, target_schema)
+ rows = []
+ batch_idx += 1
+ if rows:
+ yield Key(0, batch_idx), self._rows_to_table(rows, target_schema)
+
+ def _read_device_from_file(self, reader, meta: dict, device_key: tuple) -> tuple:
+ """Stream one device's rows from one file via ``query_table`` pushdown.
+
+ Returns ``(timestamps, {field_name: values})``. The dict only includes
+ field columns that this file owns *and* that the builder requested;
+ callers fill missing fields with all-null contributions.
+ """
+ from tsfile import tag_eq
+
+ file_tag_cols: list[str] = meta["tag_cols"]
+ file_field_cols: set[str] = meta["field_cols"]
+ time_col: str = meta["time_col"]
+
+ # Build the tag filter only over this file's tag columns. The unified
+ # device key carries one value per builder tag; map back by name.
+ unified_to_value = dict(zip(self._tag_cols, device_key))
+ tag_filter = None
+ for c in file_tag_cols:
+ v = unified_to_value.get(c)
+ if v is None:
+ # Caught earlier in _build_device_index; defensive guard.
+ return np.array([], dtype=np.int64), {}
+ expr = tag_eq(c, str(v))
+ tag_filter = expr if tag_filter is None else tag_filter & expr
+
+ # Project: requested fields ∩ this file's fields. ``query_table``
+ # always returns the time column, but it requires at least one
+ # non-time column — fall back to any owned field if the user's
+ # selection has nothing in this file.
+ requested = list(self._field_inner.keys())
+ fields_to_query = [f for f in requested if f in file_field_cols]
+ fallback_only = False
+ if not fields_to_query:
+ if not file_field_cols:
+ return np.array([], dtype=np.int64), {}
+ fields_to_query = [next(iter(file_field_cols))]
+ fallback_only = True
+
+ kwargs: dict = {"tag_filter": tag_filter, "batch_size": self.config.input_batch_size}
+ if self.config.start_time is not None:
+ kwargs["start_time"] = self.config.start_time
+ if self.config.end_time is not None:
+ kwargs["end_time"] = self.config.end_time
+
+ ts_parts: list[np.ndarray] = []
+ field_parts: dict[str, list] = {f: [] for f in fields_to_query}
+ with reader.query_table(self._table, fields_to_query, **kwargs) as rs:
+ while True:
+ batch = rs.read_arrow_batch()
+ if batch is None:
+ break
+ if batch.num_rows == 0:
+ continue
+ ts_parts.append(np.asarray(batch.column(time_col).to_numpy(), dtype=np.int64))
+ for f in fields_to_query:
+ col = batch.column(f)
+ # tsfile's arrow reader tags TIMESTAMP / DATE field columns
+ # with a fixed unit (e.g. ``timestamp[ns]``) regardless of
+ # the value's original write unit. Reinterpret as raw
+ # int64/int32 ticks so the downstream
+ # ``pa.array(type=timestamp[])`` treats them as
+ # ticks in the unit declared by our schema, instead of
+ # cross-unit casting (which would raise on data loss).
+ if pa.types.is_timestamp(col.type):
+ field_parts[f].append(col.cast(pa.int64()).to_numpy(zero_copy_only=False))
+ elif pa.types.is_date(col.type):
+ field_parts[f].append(col.cast(pa.int32()).to_numpy(zero_copy_only=False))
+ else:
+ field_parts[f].append(col.to_numpy(zero_copy_only=False))
+
+ if not ts_parts:
+ return np.array([], dtype=np.int64), {}
+ ts_full = np.concatenate(ts_parts) if len(ts_parts) > 1 else ts_parts[0]
+ vals_full = {f: (np.concatenate(parts) if len(parts) > 1 else parts[0]) for f, parts in field_parts.items()}
+
+ # Defensive boundary mask: native query paths may emit rows just
+ # outside the requested window in some chunk-boundary cases.
+ if self.config.start_time is not None or self.config.end_time is not None:
+ lo = self.config.start_time if self.config.start_time is not None else np.iinfo(np.int64).min
+ hi = self.config.end_time if self.config.end_time is not None else np.iinfo(np.int64).max
+ mask = (ts_full >= lo) & (ts_full <= hi)
+ if not mask.all():
+ ts_full = ts_full[mask]
+ vals_full = {f: arr[mask] for f, arr in vals_full.items()}
+
+ # Drop the fallback "pick one" column from the user-visible payload.
+ if fallback_only:
+ vals_full = {}
+ return ts_full, vals_full
+
+ def _finalize_device(
+ self,
+ device_key: tuple,
+ time_chunks: list,
+ field_chunks: dict,
+ field_names: list[str],
+ ) -> dict:
+ """Concatenate per-file chunks, sort by time, and return one row.
+
+ Raises ``ValueError`` if the same timestamp appears more than once
+ for a device (within or across files) — tsfile's per-device timeline
+ is required to be unique-by-timestamp.
+ """
+ time_arr = np.concatenate(time_chunks) if time_chunks else np.array([], dtype=np.int64)
+ n_total = len(time_arr)
+
+ if n_total > 0:
+ sort_idx = np.argsort(time_arr, kind="stable")
+ time_sorted = time_arr[sort_idx]
+ if n_total > 1:
+ dup_mask = time_sorted[1:] == time_sorted[:-1]
+ if dup_mask.any():
+ dup_ts = int(time_sorted[1:][dup_mask][0])
+ raise ValueError(
+ f"Duplicate timestamp {dup_ts} for device "
+ f"{dict(zip(self._tag_cols, device_key))}. "
+ "Cross-file or within-file duplicate timestamps are not supported."
+ )
+ else:
+ sort_idx = None
+ time_sorted = time_arr
+
+ row: dict = {}
+ for tag_name, tag_val in zip(self._tag_cols, device_key):
+ row[tag_name] = None if tag_val is None else str(tag_val)
+ row[self._time_col] = time_sorted
+
+ for fname in field_names:
+ chunks = field_chunks.get(fname, [])
+ materialized: list = []
+ for tchunk, fchunk in zip(time_chunks, chunks):
+ if fchunk is None:
+ materialized.append(np.full(len(tchunk), None, dtype=object))
+ else:
+ materialized.append(fchunk)
+ arr = np.concatenate(materialized) if materialized else np.array([], dtype=object)
+ if sort_idx is not None and len(arr) == n_total:
+ arr = arr[sort_idx]
+ row[fname] = arr
+ return row
+
+ # ----- arrow assembly ----------------------------------------------
+
+ def _rows_to_table(self, rows: list[dict], target_schema: pa.Schema) -> pa.Table:
+ """Convert a batch of row dicts into an Arrow table matching ``target_schema``."""
+ arrays: list[pa.Array] = []
+ for f in target_schema:
+ values = [r[f.name] for r in rows]
+ if pa.types.is_list(f.type):
+ arrays.append(self._build_list_array(values, f.type))
+ else:
+ arrays.append(pa.array(values, type=f.type))
+ pa_table = pa.Table.from_arrays(arrays, names=[f.name for f in target_schema])
+ return table_cast(pa_table, target_schema)
+
+ @staticmethod
+ def _build_list_array(values: list, list_type: pa.ListType) -> pa.ListArray:
+ """Build a ``ListArray`` from a list of per-row 1D arrays / sequences."""
+ inner_type = list_type.value_type
+ offsets = [0]
+ flat_chunks: list = []
+ total = 0
+ for v in values:
+ if v is None:
+ length = 0
+ else:
+ length = len(v)
+ flat_chunks.append(v)
+ total += length
+ offsets.append(total)
+
+ if flat_chunks:
+ try:
+ flat = np.concatenate([np.asarray(c) for c in flat_chunks])
+ except ValueError:
+ # Heterogeneous shapes / dtypes → fall back to a Python list.
+ flat = []
+ for c in flat_chunks:
+ flat.extend(list(c))
+ flat_arr = pa.array(flat, type=inner_type, from_pandas=True)
+ else:
+ flat_arr = pa.array([], type=inner_type)
+
+ return pa.ListArray.from_arrays(pa.array(offsets, type=pa.int32()), flat_arr)
+
+ # ----- file / error handling ---------------------------------------
+
+ @staticmethod
+ def _open_reader(file: str):
+ """Open a file as a ``TsFileReader`` after verifying its magic header.
+
+ The C library's ``TsFileReader`` constructor silently returns an
+ invalid handle for non-tsfile inputs, and any subsequent call on it
+ segfaults. The 6-byte ``TsFile`` magic header is checked first to
+ bail out cleanly.
+ """
+ from tsfile import TsFileReader
+
+ try:
+ with open(file, "rb") as fh:
+ header = fh.read(len(_TSFILE_MAGIC))
+ except OSError as e:
+ raise ValueError(f"Cannot open file {file!r}: {e}") from e
+ if header != _TSFILE_MAGIC:
+ raise ValueError(f"File {file!r} is not a valid TsFile (bad magic header).")
+ return TsFileReader(file)
+
+ @staticmethod
+ def _require_table_model(file: str, schemas) -> None:
+ if not schemas:
+ raise ValueError(
+ f"File {file!r} is a tree-model TsFile, which is not supported. "
+ "Only table-model TsFiles can be loaded."
+ )
+
+ @staticmethod
+ def _schemas_by_lc(reader) -> dict:
+ """Return ``get_all_table_schemas()`` keyed by lowercased table name.
+
+ TsFile / IoTDB treat table names case-insensitively, but the Python
+ binding's ``get_all_table_schemas()`` returns a dict keyed by whatever
+ casing the file was written with. Lowercasing the keys here lets all
+ downstream lookups use a single canonical form.
+ """
+ return {name.lower(): schema for name, schema in reader.get_all_table_schemas().items()}
+
+ def _should_reraise(self, file: str, exc: BaseException) -> bool:
+ """Apply ``on_bad_files`` policy. Returns True iff the caller should re-raise."""
+ mode = self.config.on_bad_files
+ if mode == "error":
+ logger.error(f"Failed to read file '{file}' with error {type(exc).__name__}: {exc}")
+ return True
+ if mode == "warn":
+ logger.warning(f"Skipping bad file '{file}'. {type(exc).__name__}: {exc}")
+ else:
+ logger.debug(f"Skipping bad file '{file}'. {type(exc).__name__}: {exc}")
+ return False
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/videofolder/__init__.py b/venv/Lib/site-packages/datasets/packaged_modules/videofolder/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/videofolder/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/videofolder/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b1eeea0ef98ce34bd3a527cb09c4838544737420
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/videofolder/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/videofolder/__pycache__/videofolder.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/videofolder/__pycache__/videofolder.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7c521dd7b53f46ecacde172b298c3e177078e6bf
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/videofolder/__pycache__/videofolder.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/videofolder/videofolder.py b/venv/Lib/site-packages/datasets/packaged_modules/videofolder/videofolder.py
new file mode 100644
index 0000000000000000000000000000000000000000..099a17a8d09e981b34c4a40a3f5e8c948d0efa5d
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/videofolder/videofolder.py
@@ -0,0 +1,34 @@
+import datasets
+
+from ..folder_based_builder import folder_based_builder
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+class VideoFolderConfig(folder_based_builder.FolderBasedBuilderConfig):
+ """BuilderConfig for ImageFolder."""
+
+ drop_labels: bool = None
+ drop_metadata: bool = None
+
+ def __post_init__(self):
+ super().__post_init__()
+
+
+class VideoFolder(folder_based_builder.FolderBasedBuilder):
+ BASE_FEATURE = datasets.Video
+ BASE_COLUMN_NAME = "video"
+ BUILDER_CONFIG_CLASS = VideoFolderConfig
+ EXTENSIONS: list[str] # definition at the bottom of the script
+
+
+# TODO: initial list, we should check the compatibility of other formats
+VIDEO_EXTENSIONS = [
+ ".mkv",
+ ".mp4",
+ ".avi",
+ ".mpeg",
+ ".mov",
+]
+VideoFolder.EXTENSIONS = VIDEO_EXTENSIONS
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/webdataset/__init__.py b/venv/Lib/site-packages/datasets/packaged_modules/webdataset/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/webdataset/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/webdataset/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..42db9ca4c84c8019e34df5c95c14f5cbf161b70f
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/webdataset/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/webdataset/__pycache__/_tenbin.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/webdataset/__pycache__/_tenbin.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..911dbc4cda997e6cfa37d5c091823dc4d829278d
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/webdataset/__pycache__/_tenbin.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/webdataset/__pycache__/webdataset.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/webdataset/__pycache__/webdataset.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..146be4b51826cd79f3e32510b7a48cc57e13ecc9
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/webdataset/__pycache__/webdataset.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/webdataset/_tenbin.py b/venv/Lib/site-packages/datasets/packaged_modules/webdataset/_tenbin.py
new file mode 100644
index 0000000000000000000000000000000000000000..cd8c054842e090dc09bdf3d2fee59241a1a928c5
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/webdataset/_tenbin.py
@@ -0,0 +1,285 @@
+#
+# Copyright (c) 2017-2021 NVIDIA CORPORATION. All rights reserved.
+# This file coems from the WebDataset library.
+# See the LICENSE file for licensing terms (BSD-style).
+#
+
+"""
+Binary tensor encodings for PyTorch and NumPy.
+
+This defines efficient binary encodings for tensors. The format is 8 byte
+aligned and can be used directly for computations when transmitted, say,
+via RDMA. The format is supported by WebDataset with the `.ten` filename
+extension. It is also used by Tensorcom, Tensorcom RDMA, and can be used
+for fast tensor storage with LMDB and in disk files (which can be memory
+mapped)
+
+Data is encoded as a series of chunks:
+
+- magic number (int64)
+- length in bytes (int64)
+- bytes (multiple of 64 bytes long)
+
+Arrays are a header chunk followed by a data chunk.
+Header chunks have the following structure:
+
+- dtype (int64)
+- 8 byte array name
+- ndim (int64)
+- dim[0]
+- dim[1]
+- ...
+"""
+
+import struct
+import sys
+
+import numpy as np
+
+
+def bytelen(a):
+ """Determine the length of a in bytes."""
+ if hasattr(a, "nbytes"):
+ return a.nbytes
+ elif isinstance(a, (bytearray, bytes)):
+ return len(a)
+ else:
+ raise ValueError(a, "cannot determine nbytes")
+
+
+def bytedata(a):
+ """Return a the raw data corresponding to a."""
+ if isinstance(a, (bytearray, bytes, memoryview)):
+ return a
+ elif hasattr(a, "data"):
+ return a.data
+ else:
+ raise ValueError(a, "cannot return bytedata")
+
+
+# tables for converting between long/short NumPy dtypes
+
+long_to_short = """
+float16 f2
+float32 f4
+float64 f8
+int8 i1
+int16 i2
+int32 i4
+int64 i8
+uint8 u1
+uint16 u2
+unit32 u4
+uint64 u8
+""".strip()
+long_to_short = [x.split() for x in long_to_short.split("\n")]
+long_to_short = {x[0]: x[1] for x in long_to_short}
+short_to_long = {v: k for k, v in long_to_short.items()}
+
+
+def check_acceptable_input_type(data, allow64):
+ """Check that the data has an acceptable type for tensor encoding.
+
+ :param data: array
+ :param allow64: allow 64 bit types
+ """
+ for a in data:
+ if a.dtype.name not in long_to_short:
+ raise ValueError("unsupported dataypte")
+ if not allow64 and a.dtype.name not in ["float64", "int64", "uint64"]:
+ raise ValueError("64 bit datatypes not allowed unless explicitly enabled")
+
+
+def str64(s):
+ """Convert a string to an int64."""
+ s = s + "\0" * (8 - len(s))
+ s = s.encode("ascii")
+ return struct.unpack("@q", s)[0]
+
+
+def unstr64(i):
+ """Convert an int64 to a string."""
+ b = struct.pack("@q", i)
+ return b.decode("ascii").strip("\0")
+
+
+def check_infos(data, infos, required_infos=None):
+ """Verify the info strings."""
+ if required_infos is False or required_infos is None:
+ return data
+ if required_infos is True:
+ return data, infos
+ if not isinstance(required_infos, (tuple, list)):
+ raise ValueError("required_infos must be tuple or list")
+ for required, actual in zip(required_infos, infos):
+ raise ValueError(f"actual info {actual} doesn't match required info {required}")
+ return data
+
+
+def encode_header(a, info=""):
+ """Encode an array header as a byte array."""
+ if a.ndim >= 10:
+ raise ValueError("too many dimensions")
+ if a.nbytes != np.prod(a.shape) * a.itemsize:
+ raise ValueError("mismatch between size and shape")
+ if a.dtype.name not in long_to_short:
+ raise ValueError("unsupported array type")
+ header = [str64(long_to_short[a.dtype.name]), str64(info), len(a.shape)] + list(a.shape)
+ return bytedata(np.array(header, dtype="i8"))
+
+
+def decode_header(h):
+ """Decode a byte array into an array header."""
+ h = np.frombuffer(h, dtype="i8")
+ if unstr64(h[0]) not in short_to_long:
+ raise ValueError("unsupported array type")
+ dtype = np.dtype(short_to_long[unstr64(h[0])])
+ info = unstr64(h[1])
+ rank = int(h[2])
+ shape = tuple(h[3 : 3 + rank])
+ return shape, dtype, info
+
+
+def encode_list(l, infos=None): # noqa: E741
+ """Given a list of arrays, encode them into a list of byte arrays."""
+ if infos is None:
+ infos = [""]
+ else:
+ if len(l) != len(infos):
+ raise ValueError(f"length of list {l} must muatch length of infos {infos}")
+ result = []
+ for i, a in enumerate(l):
+ header = encode_header(a, infos[i % len(infos)])
+ result += [header, bytedata(a)]
+ return result
+
+
+def decode_list(l, infos=False): # noqa: E741
+ """Given a list of byte arrays, decode them into arrays."""
+ result = []
+ infos0 = []
+ for header, data in zip(l[::2], l[1::2]):
+ shape, dtype, info = decode_header(header)
+ a = np.frombuffer(data, dtype=dtype, count=np.prod(shape)).reshape(*shape)
+ result += [a]
+ infos0 += [info]
+ return check_infos(result, infos0, infos)
+
+
+magic_str = "~TenBin~"
+magic = str64(magic_str)
+magic_bytes = unstr64(magic).encode("ascii")
+
+
+def roundup(n, k=64):
+ """Round up to the next multiple of 64."""
+ return k * ((n + k - 1) // k)
+
+
+def encode_chunks(l): # noqa: E741
+ """Encode a list of chunks into a single byte array, with lengths and magics.."""
+ size = sum(16 + roundup(b.nbytes) for b in l)
+ result = bytearray(size)
+ offset = 0
+ for b in l:
+ result[offset : offset + 8] = magic_bytes
+ offset += 8
+ result[offset : offset + 8] = struct.pack("@q", b.nbytes)
+ offset += 8
+ result[offset : offset + bytelen(b)] = b
+ offset += roundup(bytelen(b))
+ return result
+
+
+def decode_chunks(buf):
+ """Decode a byte array into a list of chunks."""
+ result = []
+ offset = 0
+ total = bytelen(buf)
+ while offset < total:
+ if magic_bytes != buf[offset : offset + 8]:
+ raise ValueError("magic bytes mismatch")
+ offset += 8
+ nbytes = struct.unpack("@q", buf[offset : offset + 8])[0]
+ offset += 8
+ b = buf[offset : offset + nbytes]
+ offset += roundup(nbytes)
+ result.append(b)
+ return result
+
+
+def encode_buffer(l, infos=None): # noqa: E741
+ """Encode a list of arrays into a single byte array."""
+ if not isinstance(l, list):
+ raise ValueError("requires list")
+ return encode_chunks(encode_list(l, infos=infos))
+
+
+def decode_buffer(buf, infos=False):
+ """Decode a byte array into a list of arrays."""
+ return decode_list(decode_chunks(buf), infos=infos)
+
+
+def write_chunk(stream, buf):
+ """Write a byte chunk to the stream with magics, length, and padding."""
+ nbytes = bytelen(buf)
+ stream.write(magic_bytes)
+ stream.write(struct.pack("@q", nbytes))
+ stream.write(bytedata(buf))
+ padding = roundup(nbytes) - nbytes
+ if padding > 0:
+ stream.write(b"\0" * padding)
+
+
+def read_chunk(stream):
+ """Read a byte chunk from a stream with magics, length, and padding."""
+ magic = stream.read(8)
+ if magic == b"":
+ return None
+ if magic != magic_bytes:
+ raise ValueError("magic number does not match")
+ nbytes = stream.read(8)
+ nbytes = struct.unpack("@q", nbytes)[0]
+ if nbytes < 0:
+ raise ValueError("negative nbytes")
+ data = stream.read(nbytes)
+ padding = roundup(nbytes) - nbytes
+ if padding > 0:
+ stream.read(padding)
+ return data
+
+
+def write(stream, l, infos=None): # noqa: E741
+ """Write a list of arrays to a stream, with magics, length, and padding."""
+ for chunk in encode_list(l, infos=infos):
+ write_chunk(stream, chunk)
+
+
+def read(stream, n=sys.maxsize, infos=False):
+ """Read a list of arrays from a stream, with magics, length, and padding."""
+ chunks = []
+ for _ in range(n):
+ header = read_chunk(stream)
+ if header is None:
+ break
+ data = read_chunk(stream)
+ if data is None:
+ raise ValueError("premature EOF")
+ chunks += [header, data]
+ return decode_list(chunks, infos=infos)
+
+
+def save(fname, *args, infos=None, nocheck=False):
+ """Save a list of arrays to a file, with magics, length, and padding."""
+ if not nocheck and not fname.endswith(".ten"):
+ raise ValueError("file name should end in .ten")
+ with open(fname, "wb") as stream:
+ write(stream, args, infos=infos)
+
+
+def load(fname, infos=False, nocheck=False):
+ """Read a list of arrays from a file, with magics, length, and padding."""
+ if not nocheck and not fname.endswith(".ten"):
+ raise ValueError("file name should end in .ten")
+ with open(fname, "rb") as stream:
+ return read(stream, infos=infos)
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/webdataset/webdataset.py b/venv/Lib/site-packages/datasets/packaged_modules/webdataset/webdataset.py
new file mode 100644
index 0000000000000000000000000000000000000000..4c8c21b17cdece93bb15db9e94d57c0c63bc4ce7
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/webdataset/webdataset.py
@@ -0,0 +1,367 @@
+import io
+import json
+import re
+from itertools import islice
+from typing import Any, Callable
+
+import fsspec
+import numpy as np
+import pyarrow as pa
+
+import datasets
+from datasets.builder import Key
+from datasets.features.features import cast_to_python_objects
+from datasets.utils.file_utils import SINGLE_FILE_COMPRESSION_EXTENSION_TO_PROTOCOL, xbasename
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+class WebDataset(datasets.GeneratorBasedBuilder):
+ DEFAULT_WRITER_BATCH_SIZE = 100
+ IMAGE_EXTENSIONS: list[str] # definition at the bottom of the script
+ AUDIO_EXTENSIONS: list[str] # definition at the bottom of the script
+ VIDEO_EXTENSIONS: list[str] # definition at the bottom of the script
+ MESH_EXTENSIONS: list[str] # definition at the bottom of the script
+ DECODERS: dict[str, Callable[[Any], Any]] # definition at the bottom of the script
+ NUM_EXAMPLES_FOR_FEATURES_INFERENCE = 5
+
+ @classmethod
+ def _get_pipeline_from_tar(cls, tar_path, tar_iterator):
+ current_example = {}
+ fs: fsspec.AbstractFileSystem = fsspec.filesystem("memory")
+ streaming_download_manager = datasets.StreamingDownloadManager()
+ for filename, f in tar_iterator:
+ example_key, field_name = base_plus_ext(filename)
+ if example_key is None:
+ continue
+ if current_example and current_example["__key__"] != example_key:
+ # reposition some keys in last position
+ current_example["__key__"] = current_example.pop("__key__")
+ current_example["__url__"] = current_example.pop("__url__")
+ yield current_example
+ current_example = {}
+ current_example["__key__"] = example_key
+ current_example["__url__"] = tar_path
+ current_example[field_name] = f.read()
+ if field_name.split(".")[-1].lower() in SINGLE_FILE_COMPRESSION_EXTENSION_TO_PROTOCOL:
+ fs.write_bytes(filename, current_example[field_name])
+ extracted_file_path = streaming_download_manager.extract(f"memory://{filename}")
+ with fsspec.open(extracted_file_path) as f:
+ current_example[field_name] = f.read()
+ fs.delete(filename)
+ data_extension = xbasename(extracted_file_path).split(".")[-1].lower()
+ else:
+ data_extension = field_name.split(".")[-1].lower()
+ if data_extension in cls.DECODERS:
+ current_example[field_name] = cls.DECODERS[data_extension](current_example[field_name])
+ if current_example:
+ yield current_example
+
+ def _info(self) -> datasets.DatasetInfo:
+ return datasets.DatasetInfo()
+
+ def _split_generators(self, dl_manager):
+ """We handle string, list and dicts in datafiles"""
+ # Download the data files
+ if not self.config.data_files:
+ raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+ data_files = dl_manager.download(self.config.data_files)
+ splits = []
+ for split_name, tar_paths in data_files.items():
+ tar_iterators = [dl_manager.iter_archive(tar_path) for tar_path in tar_paths]
+ splits.append(
+ datasets.SplitGenerator(
+ name=split_name, gen_kwargs={"tar_paths": tar_paths, "tar_iterators": tar_iterators}
+ )
+ )
+ if not self.info.features:
+ # Get one example to get the feature types
+ pipeline = self._get_pipeline_from_tar(tar_paths[0], tar_iterators[0])
+ first_examples = list(islice(pipeline, self.NUM_EXAMPLES_FOR_FEATURES_INFERENCE))
+ if any(example.keys() != first_examples[0].keys() for example in first_examples):
+ raise ValueError(
+ "The TAR archives of the dataset should be in WebDataset format, "
+ "but the files in the archive don't share the same prefix or the same types."
+ )
+ pa_tables = [
+ pa.Table.from_pylist(cast_to_python_objects([example], only_1d_for_numpy=True))
+ for example in first_examples
+ ]
+ inferred_arrow_schema = pa.concat_tables(pa_tables, promote_options="default").schema
+ features = datasets.Features.from_arrow_schema(inferred_arrow_schema)
+
+ for field_name in first_examples[0]:
+ extension = field_name.rsplit(".", 1)[-1].lower()
+ # Set Image types
+ if extension in self.IMAGE_EXTENSIONS:
+ features[field_name] = datasets.Image()
+ # Set Audio types
+ if extension in self.AUDIO_EXTENSIONS:
+ features[field_name] = datasets.Audio()
+ # Set Video types
+ if extension in self.VIDEO_EXTENSIONS:
+ features[field_name] = datasets.Video()
+ # Set Mesh types
+ if extension in self.MESH_EXTENSIONS:
+ features[field_name] = datasets.Mesh()
+ self.info.features = features
+
+ return splits
+
+ def _generate_shards(self, tar_paths, tar_iterators):
+ yield from tar_paths
+
+ def _generate_examples(self, tar_paths, tar_iterators):
+ image_field_names = [
+ field_name for field_name, feature in self.info.features.items() if isinstance(feature, datasets.Image)
+ ]
+ audio_field_names = [
+ field_name for field_name, feature in self.info.features.items() if isinstance(feature, datasets.Audio)
+ ]
+ video_field_names = [
+ field_name for field_name, feature in self.info.features.items() if isinstance(feature, datasets.Video)
+ ]
+ mesh_field_names = [
+ field_name for field_name, feature in self.info.features.items() if isinstance(feature, datasets.Mesh)
+ ]
+ all_field_names = list(self.info.features.keys())
+ for tar_idx, (tar_path, tar_iterator) in enumerate(zip(tar_paths, tar_iterators)):
+ for example_idx, example in enumerate(self._get_pipeline_from_tar(tar_path, tar_iterator)):
+ for field_name in all_field_names:
+ if field_name not in example:
+ example[field_name] = None
+ for field_name in image_field_names + audio_field_names + video_field_names + mesh_field_names:
+ if example[field_name] is not None:
+ example[field_name] = {
+ "path": example["__key__"] + "." + field_name,
+ "bytes": example[field_name],
+ }
+ yield Key(tar_idx, example_idx), example
+
+
+# Source: https://github.com/webdataset/webdataset/blob/87bd5aa41602d57f070f65a670893ee625702f2f/webdataset/tariterators.py#L25
+def base_plus_ext(path):
+ """Split off all file extensions.
+
+ Returns base, allext.
+ """
+ match = re.match(r"^((?:.*/|)[^.]+)[.]([^/]*)$", path)
+ if not match:
+ return None, None
+ return match.group(1), match.group(2)
+
+
+# Obtained with:
+# ```
+# import PIL.Image
+# IMAGE_EXTENSIONS = []
+# PIL.Image.init()
+# for ext, format in PIL.Image.EXTENSION.items():
+# if format in PIL.Image.OPEN:
+# IMAGE_EXTENSIONS.append(ext[1:])
+# ```
+# We intentionally do not run this code on launch because:
+# (1) Pillow is an optional dependency, so importing Pillow in global namespace is not allowed
+# (2) To ensure the list of supported extensions is deterministic
+IMAGE_EXTENSIONS = [
+ "blp",
+ "bmp",
+ "dib",
+ "bufr",
+ "cur",
+ "pcx",
+ "dcx",
+ "dds",
+ "ps",
+ "eps",
+ "fit",
+ "fits",
+ "fli",
+ "flc",
+ "ftc",
+ "ftu",
+ "gbr",
+ "gif",
+ "grib",
+ "h5",
+ "hdf",
+ "png",
+ "apng",
+ "jp2",
+ "j2k",
+ "jpc",
+ "jpf",
+ "jpx",
+ "j2c",
+ "icns",
+ "ico",
+ "im",
+ "iim",
+ "tif",
+ "tiff",
+ "jfif",
+ "jpe",
+ "jpg",
+ "jpeg",
+ "mpg",
+ "mpeg",
+ "msp",
+ "pcd",
+ "pxr",
+ "pbm",
+ "pgm",
+ "ppm",
+ "pnm",
+ "psd",
+ "bw",
+ "rgb",
+ "rgba",
+ "sgi",
+ "ras",
+ "tga",
+ "icb",
+ "vda",
+ "vst",
+ "webp",
+ "wmf",
+ "emf",
+ "xbm",
+ "xpm",
+]
+WebDataset.IMAGE_EXTENSIONS = IMAGE_EXTENSIONS
+
+
+# Obtained with:
+# ```
+# import soundfile as sf
+#
+# AUDIO_EXTENSIONS = [f".{format.lower()}" for format in sf.available_formats().keys()]
+#
+# # .opus decoding is supported if libsndfile >= 1.0.31:
+# AUDIO_EXTENSIONS.extend([".mp3", ".opus"])
+# ```
+# We intentionally did not run this code on launch because:
+# (1) Soundfile was an optional dependency, so importing it in global namespace is not allowed
+# (2) To ensure the list of supported extensions is deterministic
+# (3) We use TorchCodec now anyways instead of Soundfile
+AUDIO_EXTENSIONS = [
+ "aiff",
+ "au",
+ "avr",
+ "caf",
+ "flac",
+ "htk",
+ "svx",
+ "mat4",
+ "mat5",
+ "mpc2k",
+ "ogg",
+ "paf",
+ "pvf",
+ "raw",
+ "rf64",
+ "sd2",
+ "sds",
+ "ircam",
+ "voc",
+ "w64",
+ "wav",
+ "nist",
+ "wavex",
+ "wve",
+ "xi",
+ "mp3",
+ "opus",
+]
+WebDataset.AUDIO_EXTENSIONS = AUDIO_EXTENSIONS
+
+
+# TODO: initial list, we should check the compatibility of other formats
+VIDEO_EXTENSIONS = [
+ "mkv",
+ "mp4",
+ "avi",
+ "mpeg",
+ "mov",
+]
+WebDataset.VIDEO_EXTENSIONS = VIDEO_EXTENSIONS
+
+
+MESH_EXTENSIONS = [
+ "glb",
+ "ply",
+ "stl",
+]
+WebDataset.MESH_EXTENSIONS = MESH_EXTENSIONS
+
+
+def text_loads(data: bytes):
+ return data.decode("utf-8")
+
+
+def tenbin_loads(data: bytes):
+ from . import _tenbin
+
+ return _tenbin.decode_buffer(data)
+
+
+def msgpack_loads(data: bytes):
+ import msgpack
+
+ return msgpack.unpackb(data)
+
+
+def npy_loads(data: bytes):
+ import numpy.lib.format
+
+ stream = io.BytesIO(data)
+ return numpy.lib.format.read_array(stream, allow_pickle=False)
+
+
+def npz_loads(data: bytes):
+ return np.load(io.BytesIO(data), allow_pickle=False)
+
+
+def cbor_loads(data: bytes):
+ import cbor
+
+ return cbor.loads(data)
+
+
+def torch_loads(data: bytes):
+ import torch
+
+ return torch.load(io.BytesIO(data), weights_only=True)
+
+
+# Obtained by checking `decoders` in `webdataset.autodecode`
+# and removing unsafe extension decoders.
+# Removed Pickle decoders:
+# - "pyd": lambda data: pickle.loads(data)
+# - "pickle": lambda data: pickle.loads(data)
+# Modified NumPy decoders to fix CVE-2019-6446 (add allow_pickle=False and weights_only=True):
+# - "npy": npy_loads,
+# - "npz": lambda data: np.load(io.BytesIO(data)),
+# - "pth": lambda data: torch_loads(data)
+DECODERS = {
+ "txt": text_loads,
+ "text": text_loads,
+ "transcript": text_loads,
+ "cls": int,
+ "cls2": int,
+ "index": int,
+ "inx": int,
+ "id": int,
+ "json": json.loads,
+ "jsn": json.loads,
+ "ten": tenbin_loads,
+ "tb": tenbin_loads,
+ "mp": msgpack_loads,
+ "msg": msgpack_loads,
+ "npy": npy_loads,
+ "npz": npz_loads,
+ "cbor": cbor_loads,
+ "pth": torch_loads,
+}
+WebDataset.DECODERS = DECODERS
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/xml/__init__.py b/venv/Lib/site-packages/datasets/packaged_modules/xml/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/xml/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/xml/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..83d5a0be76f1e67cd33728178ba170593346dd3a
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/xml/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/xml/__pycache__/xml.cpython-311.pyc b/venv/Lib/site-packages/datasets/packaged_modules/xml/__pycache__/xml.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ed5cba87eb9a3be86ce58653558c1562bf8aaf42
Binary files /dev/null and b/venv/Lib/site-packages/datasets/packaged_modules/xml/__pycache__/xml.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/packaged_modules/xml/xml.py b/venv/Lib/site-packages/datasets/packaged_modules/xml/xml.py
new file mode 100644
index 0000000000000000000000000000000000000000..769ca7c6e0094a16d297b1c0976facedf4a35801
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/packaged_modules/xml/xml.py
@@ -0,0 +1,70 @@
+from dataclasses import dataclass
+from typing import Optional
+
+import pyarrow as pa
+
+import datasets
+from datasets.features.features import require_storage_cast
+from datasets.table import table_cast
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+@dataclass
+class XmlConfig(datasets.BuilderConfig):
+ """BuilderConfig for xml files."""
+
+ features: Optional[datasets.Features] = None
+ encoding: str = "utf-8"
+ encoding_errors: Optional[str] = None
+
+
+class Xml(datasets.ArrowBasedBuilder):
+ BUILDER_CONFIG_CLASS = XmlConfig
+
+ def _info(self):
+ return datasets.DatasetInfo(features=self.config.features)
+
+ def _split_generators(self, dl_manager):
+ """The `data_files` kwarg in load_dataset() can be a str, List[str], Dict[str,str], or Dict[str,List[str]].
+
+ If str or List[str], then the dataset returns only the 'train' split.
+ If dict, then keys should be from the `datasets.Split` enum.
+ """
+ if not self.config.data_files:
+ raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+ dl_manager.download_config.extract_on_the_fly = True
+ data_files = dl_manager.download_and_extract(self.config.data_files)
+ splits = []
+ for split_name, files in data_files.items():
+ if isinstance(files, str):
+ files = [files]
+ files = [dl_manager.iter_files(file) for file in files]
+ splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files}))
+ return splits
+
+ def _cast_table(self, pa_table: pa.Table) -> pa.Table:
+ if self.config.features is not None:
+ schema = self.config.features.arrow_schema
+ if all(not require_storage_cast(feature) for feature in self.config.features.values()):
+ # cheaper cast
+ pa_table = pa_table.cast(schema)
+ else:
+ # more expensive cast; allows str <-> int/float or str to Audio for example
+ pa_table = table_cast(pa_table, schema)
+ return pa_table
+ else:
+ return pa_table.cast(pa.schema({"xml": pa.string()}))
+
+ def _generate_shards(self, files):
+ yield from files
+
+ def _generate_tables(self, files):
+ pa_table_names = list(self.config.features) if self.config.features is not None else ["xml"]
+ for file_idx, file in enumerate(files):
+ # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
+ with open(file, encoding=self.config.encoding, errors=self.config.encoding_errors) as f:
+ xml = f.read()
+ pa_table = pa.Table.from_arrays([pa.array([xml])], names=pa_table_names)
+ yield (file_idx, 0), self._cast_table(pa_table)
diff --git a/venv/Lib/site-packages/datasets/parallel/__init__.py b/venv/Lib/site-packages/datasets/parallel/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..a8d527f4d09946b3b7b7a65ec22afa953ddfd315
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/parallel/__init__.py
@@ -0,0 +1 @@
+from .parallel import ParallelBackendConfig, parallel_backend, parallel_map
diff --git a/venv/Lib/site-packages/datasets/parallel/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/parallel/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..5b69a851283deaab98b2eb104c85db8e045036da
Binary files /dev/null and b/venv/Lib/site-packages/datasets/parallel/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/parallel/__pycache__/parallel.cpython-311.pyc b/venv/Lib/site-packages/datasets/parallel/__pycache__/parallel.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ced04a1cc845e793bc809d0480e683b166480061
Binary files /dev/null and b/venv/Lib/site-packages/datasets/parallel/__pycache__/parallel.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/parallel/parallel.py b/venv/Lib/site-packages/datasets/parallel/parallel.py
new file mode 100644
index 0000000000000000000000000000000000000000..5cad2c48ba2b0a6e71c62babd4eb052a3dd1dfd4
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/parallel/parallel.py
@@ -0,0 +1,120 @@
+import contextlib
+from multiprocessing import Pool, RLock
+
+from tqdm.auto import tqdm
+
+from ..utils import experimental, logging
+
+
+logger = logging.get_logger(__name__)
+
+
+class ParallelBackendConfig:
+ backend_name = None
+
+
+@experimental
+def parallel_map(function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, single_map_nested_func):
+ """
+ **Experimental.** Apply a function to iterable elements in parallel, where the implementation uses either
+ multiprocessing.Pool or joblib for parallelization.
+
+ Args:
+ function (`Callable[[Any], Any]`): Function to be applied to `iterable`.
+ iterable (`list`, `tuple` or `np.ndarray`): Iterable elements to apply function to.
+ num_proc (`int`): Number of processes (if no backend specified) or jobs (using joblib).
+ types (`tuple`): Additional types (besides `dict` values) to apply `function` recursively to their elements.
+ disable_tqdm (`bool`): Whether to disable the tqdm progressbar.
+ desc (`str`): Prefix for the tqdm progressbar.
+ single_map_nested_func (`Callable`): Map function that applies `function` to an element from `iterable`.
+ Takes a tuple of function, data_struct, types, rank, disable_tqdm, desc as input, where data_struct is an
+ element of `iterable`, and `rank` is used for progress bar.
+ """
+ if ParallelBackendConfig.backend_name is None:
+ return _map_with_multiprocessing_pool(
+ function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, single_map_nested_func
+ )
+
+ return _map_with_joblib(
+ function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, single_map_nested_func
+ )
+
+
+def _map_with_multiprocessing_pool(
+ function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, single_map_nested_func
+):
+ num_proc = num_proc if num_proc <= len(iterable) else len(iterable)
+ split_kwds = [] # We organize the splits ourselve (contiguous splits)
+ for index in range(num_proc):
+ div = len(iterable) // num_proc
+ mod = len(iterable) % num_proc
+ start = div * index + min(index, mod)
+ end = start + div + (1 if index < mod else 0)
+ split_kwds.append((function, iterable[start:end], batched, batch_size, types, index, disable_tqdm, desc))
+
+ if len(iterable) != sum(len(i[1]) for i in split_kwds):
+ raise ValueError(
+ f"Error dividing inputs iterable among processes. "
+ f"Total number of objects {len(iterable)}, "
+ f"length: {sum(len(i[1]) for i in split_kwds)}"
+ )
+
+ logger.info(
+ f"Spawning {num_proc} processes for {len(iterable)} objects in slices of {[len(i[1]) for i in split_kwds]}"
+ )
+ initargs, initializer = None, None
+ if not disable_tqdm:
+ initargs, initializer = (RLock(),), tqdm.set_lock
+ with Pool(num_proc, initargs=initargs, initializer=initializer) as pool:
+ mapped = pool.map(single_map_nested_func, split_kwds)
+ logger.info(f"Finished {num_proc} processes")
+ mapped = [obj for proc_res in mapped for obj in proc_res]
+ logger.info(f"Unpacked {len(mapped)} objects")
+
+ return mapped
+
+
+def _map_with_joblib(
+ function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, single_map_nested_func
+):
+ # progress bar is not yet supported for _map_with_joblib, because tqdm couldn't accurately be applied to joblib,
+ # and it requires monkey-patching joblib internal classes which is subject to change
+ import joblib
+
+ with joblib.parallel_backend(ParallelBackendConfig.backend_name, n_jobs=num_proc):
+ return joblib.Parallel()(
+ joblib.delayed(single_map_nested_func)((function, obj, batched, batch_size, types, None, True, None))
+ for obj in iterable
+ )
+
+
+@experimental
+@contextlib.contextmanager
+def parallel_backend(backend_name: str):
+ """
+ **Experimental.** Configures the parallel backend for parallelized dataset loading, which uses the parallelization
+ implemented by joblib.
+
+ Args:
+ backend_name (str): Name of backend for parallelization implementation, has to be supported by joblib.
+
+ Example usage:
+ ```py
+ with parallel_backend('spark'):
+ dataset = load_dataset(..., num_proc=2)
+ ```
+ """
+ ParallelBackendConfig.backend_name = backend_name
+
+ if backend_name == "spark":
+ from joblibspark import register_spark
+
+ register_spark()
+
+ # TODO: call create_cache_and_write_probe if "download" in steps
+ # TODO: raise NotImplementedError when Dataset.map etc is called
+
+ try:
+ yield
+ finally:
+ ParallelBackendConfig.backend_name = None
diff --git a/venv/Lib/site-packages/datasets/search.py b/venv/Lib/site-packages/datasets/search.py
new file mode 100644
index 0000000000000000000000000000000000000000..cb994d24e41e96174386de79057441e4436cc19a
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/search.py
@@ -0,0 +1,785 @@
+import importlib.util
+import os
+import tempfile
+from pathlib import PurePath
+from typing import TYPE_CHECKING, NamedTuple, Optional, Union
+
+import fsspec
+import numpy as np
+
+from .features import List
+from .utils import logging
+from .utils import tqdm as hf_tqdm
+
+
+if TYPE_CHECKING:
+ from .arrow_dataset import Dataset # noqa: F401
+
+ try:
+ from elasticsearch import Elasticsearch # noqa: F401
+
+ except ImportError:
+ pass
+ try:
+ import faiss # noqa: F401
+
+ except ImportError:
+ pass
+
+_has_elasticsearch = importlib.util.find_spec("elasticsearch") is not None
+_has_faiss = importlib.util.find_spec("faiss") is not None
+
+
+logger = logging.get_logger(__name__)
+
+
+class MissingIndex(Exception):
+ pass
+
+
+class SearchResults(NamedTuple):
+ scores: list[float]
+ indices: list[int]
+
+
+class BatchedSearchResults(NamedTuple):
+ total_scores: list[list[float]]
+ total_indices: list[list[int]]
+
+
+class NearestExamplesResults(NamedTuple):
+ scores: list[float]
+ examples: dict
+
+
+class BatchedNearestExamplesResults(NamedTuple):
+ total_scores: list[list[float]]
+ total_examples: list[dict]
+
+
+class BaseIndex:
+ """Base class for indexing"""
+
+ def search(self, query, k: int = 10, **kwargs) -> SearchResults:
+ """
+ To implement.
+ This method has to return the scores and the indices of the retrieved examples given a certain query.
+ """
+ raise NotImplementedError
+
+ def search_batch(self, queries, k: int = 10, **kwargs) -> BatchedSearchResults:
+ """Find the nearest examples indices to the query.
+
+ Args:
+ queries (`Union[List[str], np.ndarray]`): The queries as a list of strings if `column` is a text index or as a numpy array if `column` is a vector index.
+ k (`int`): The number of examples to retrieve per query.
+
+ Output:
+ total_scores (`List[List[float]`): The retrieval scores of the retrieved examples per query.
+ total_indices (`List[List[int]]`): The indices of the retrieved examples per query.
+ """
+ total_scores, total_indices = [], []
+ for query in queries:
+ scores, indices = self.search(query, k)
+ total_scores.append(scores)
+ total_indices.append(indices)
+ return BatchedSearchResults(total_scores, total_indices)
+
+ def save(self, file: Union[str, PurePath]):
+ """Serialize the index on disk"""
+ raise NotImplementedError
+
+ @classmethod
+ def load(cls, file: Union[str, PurePath]) -> "BaseIndex":
+ """Deserialize the index from disk"""
+ raise NotImplementedError
+
+
+class ElasticSearchIndex(BaseIndex):
+ """
+ Sparse index using Elasticsearch. It is used to index text and run queries based on BM25 similarity.
+ An Elasticsearch server needs to be accessible, and a python client is declared with
+ ```
+ es_client = Elasticsearch([{'host': 'localhost', 'port': '9200'}])
+ ```
+ for example.
+ """
+
+ def __init__(
+ self,
+ host: Optional[str] = None,
+ port: Optional[int] = None,
+ es_client: Optional["Elasticsearch"] = None,
+ es_index_name: Optional[str] = None,
+ es_index_config: Optional[dict] = None,
+ ):
+ if not _has_elasticsearch:
+ raise ImportError(
+ "You must install ElasticSearch to use ElasticSearchIndex. To do so you can run `pip install elasticsearch==7.7.1 for example`"
+ )
+ if es_client is not None and (host is not None or port is not None):
+ raise ValueError("Please specify either `es_client` or `(host, port)`, but not both.")
+ host = host or "localhost"
+ port = port or 9200
+
+ import elasticsearch.helpers # noqa: F401 - need this to properly load all the es features
+ from elasticsearch import Elasticsearch # noqa: F811
+
+ self.es_client = es_client if es_client is not None else Elasticsearch([{"host": host, "port": str(port)}])
+ self.es_index_name = (
+ es_index_name
+ if es_index_name is not None
+ else "huggingface_datasets_" + os.path.basename(tempfile.NamedTemporaryFile().name)
+ )
+ self.es_index_config = (
+ es_index_config
+ if es_index_config is not None
+ else {
+ "settings": {
+ "number_of_shards": 1,
+ "analysis": {"analyzer": {"stop_standard": {"type": "standard", " stopwords": "_english_"}}},
+ },
+ "mappings": {"properties": {"text": {"type": "text", "analyzer": "standard", "similarity": "BM25"}}},
+ }
+ )
+
+ def add_documents(self, documents: Union[list[str], "Dataset"], column: Optional[str] = None):
+ """
+ Add documents to the index.
+ If the documents are inside a certain column, you can specify it using the `column` argument.
+ """
+ index_name = self.es_index_name
+ index_config = self.es_index_config
+ self.es_client.indices.create(index=index_name, body=index_config)
+ number_of_docs = len(documents)
+ progress = hf_tqdm(unit="docs", total=number_of_docs)
+ successes = 0
+
+ def passage_generator():
+ if column is not None:
+ for i, example in enumerate(documents):
+ yield {"text": example[column], "_id": i}
+ else:
+ for i, example in enumerate(documents):
+ yield {"text": example, "_id": i}
+
+ # create the ES index
+ import elasticsearch as es
+
+ for ok, action in es.helpers.streaming_bulk(
+ client=self.es_client,
+ index=index_name,
+ actions=passage_generator(),
+ ):
+ progress.update(1)
+ successes += ok
+ if successes != len(documents):
+ logger.warning(
+ f"Some documents failed to be added to ElasticSearch. Failures: {len(documents) - successes}/{len(documents)}"
+ )
+ logger.info(f"Indexed {successes:d} documents")
+
+ def search(self, query: str, k=10, **kwargs) -> SearchResults:
+ """Find the nearest examples indices to the query.
+
+ Args:
+ query (`str`): The query as a string.
+ k (`int`): The number of examples to retrieve.
+
+ Output:
+ scores (`List[List[float]`): The retrieval scores of the retrieved examples.
+ indices (`List[List[int]]`): The indices of the retrieved examples.
+ """
+ response = self.es_client.search(
+ index=self.es_index_name,
+ body={"query": {"multi_match": {"query": query, "fields": ["text"], "type": "cross_fields"}}, "size": k},
+ **kwargs,
+ )
+ hits = response["hits"]["hits"]
+ return SearchResults([hit["_score"] for hit in hits], [int(hit["_id"]) for hit in hits])
+
+ def search_batch(self, queries, k: int = 10, max_workers=10, **kwargs) -> BatchedSearchResults:
+ import concurrent.futures
+
+ total_scores, total_indices = [None] * len(queries), [None] * len(queries)
+ with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+ future_to_index = {executor.submit(self.search, query, k, **kwargs): i for i, query in enumerate(queries)}
+ for future in concurrent.futures.as_completed(future_to_index):
+ index = future_to_index[future]
+ results: SearchResults = future.result()
+ total_scores[index] = results.scores
+ total_indices[index] = results.indices
+ return BatchedSearchResults(total_indices=total_indices, total_scores=total_scores)
+
+
+class FaissIndex(BaseIndex):
+ """
+ Dense index using Faiss. It is used to index vectors.
+ Faiss is a library for efficient similarity search and clustering of dense vectors.
+ It contains algorithms that search in sets of vectors of any size, up to ones that possibly do not fit in RAM.
+ You can find more information about Faiss here:
+ - For index types and the string factory: https://github.com/facebookresearch/faiss/wiki/The-index-factory
+ - For GPU settings: https://github.com/facebookresearch/faiss/wiki/Faiss-on-the-GPU
+ """
+
+ def __init__(
+ self,
+ device: Optional[Union[int, list[int]]] = None,
+ string_factory: Optional[str] = None,
+ metric_type: Optional[int] = None,
+ custom_index: Optional["faiss.Index"] = None,
+ ):
+ """
+ Create a Dense index using Faiss. You can specify `device` if you want to run it on GPU (`device` must be the GPU index).
+ You can find more information about Faiss here:
+ - For `string factory`: https://github.com/facebookresearch/faiss/wiki/The-index-factory
+ """
+ if string_factory is not None and custom_index is not None:
+ raise ValueError("Please specify either `string_factory` or `custom_index` but not both.")
+ if device is not None and custom_index is not None:
+ raise ValueError(
+ "Cannot pass both 'custom_index' and 'device'. "
+ "Pass 'custom_index' already transferred to the target device instead."
+ )
+ self.device = device
+ self.string_factory = string_factory
+ self.metric_type = metric_type
+ self.faiss_index = custom_index
+ if not _has_faiss:
+ raise ImportError(
+ "You must install Faiss to use FaissIndex. To do so you can run `conda install -c pytorch faiss-cpu` or `conda install -c pytorch faiss-gpu`. "
+ "A community supported package is also available on pypi: `pip install faiss-cpu` or `pip install faiss-gpu`. "
+ "Note that pip may not have the latest version of FAISS, and thus, some of the latest features and bug fixes may not be available."
+ )
+
+ def add_vectors(
+ self,
+ vectors: Union[np.array, "Dataset"],
+ column: Optional[str] = None,
+ batch_size: int = 1000,
+ train_size: Optional[int] = None,
+ faiss_verbose: Optional[bool] = None,
+ ):
+ """
+ Add vectors to the index.
+ If the arrays are inside a certain column, you can specify it using the `column` argument.
+ """
+ import faiss # noqa: F811
+
+ if column and not isinstance(vectors.features[column], List):
+ raise ValueError(
+ f"Wrong feature type for column '{column}'. Expected 1d array, got {vectors.features[column]}"
+ )
+
+ # Create index
+ if self.faiss_index is None:
+ size = len(vectors[0]) if column is None else len(vectors[0][column])
+ if self.string_factory is not None:
+ if self.metric_type is None:
+ index = faiss.index_factory(size, self.string_factory)
+ else:
+ index = faiss.index_factory(size, self.string_factory, self.metric_type)
+ else:
+ if self.metric_type is None:
+ index = faiss.IndexFlat(size)
+ else:
+ index = faiss.IndexFlat(size, self.metric_type)
+
+ self.faiss_index = self._faiss_index_to_device(index, self.device)
+ logger.info(f"Created faiss index of type {type(self.faiss_index)}")
+
+ # Set verbosity level
+ if faiss_verbose is not None:
+ self.faiss_index.verbose = faiss_verbose
+ if hasattr(self.faiss_index, "index") and self.faiss_index.index is not None:
+ self.faiss_index.index.verbose = faiss_verbose
+ if hasattr(self.faiss_index, "quantizer") and self.faiss_index.quantizer is not None:
+ self.faiss_index.quantizer.verbose = faiss_verbose
+ if hasattr(self.faiss_index, "clustering_index") and self.faiss_index.clustering_index is not None:
+ self.faiss_index.clustering_index.verbose = faiss_verbose
+
+ # Train
+ if train_size is not None:
+ train_vecs = vectors[:train_size] if column is None else vectors[:train_size][column]
+ logger.info(f"Training the index with the first {len(train_vecs)} vectors")
+ self.faiss_index.train(train_vecs)
+ else:
+ logger.info("Ignored the training step of the faiss index as `train_size` is None.")
+
+ # Add vectors
+ logger.info(f"Adding {len(vectors)} vectors to the faiss index")
+ for i in hf_tqdm(range(0, len(vectors), batch_size)):
+ vecs = vectors[i : i + batch_size] if column is None else vectors[i : i + batch_size][column]
+ self.faiss_index.add(vecs)
+
+ @staticmethod
+ def _faiss_index_to_device(index: "faiss.Index", device: Optional[Union[int, list[int]]] = None) -> "faiss.Index":
+ """
+ Sends a faiss index to a device.
+ A device can either be a positive integer (GPU id), a negative integer (all GPUs),
+ or a list of positive integers (select GPUs to use), or `None` for CPU.
+ """
+
+ # If device is not specified, then it runs on CPU.
+ if device is None:
+ return index
+
+ import faiss # noqa: F811
+
+ # If the device id is given as an integer
+ if isinstance(device, int):
+ # Positive integers are directly mapped to GPU ids
+ if device > -1:
+ faiss_res = faiss.StandardGpuResources()
+ index = faiss.index_cpu_to_gpu(faiss_res, device, index)
+ # And negative integers mean using all GPUs
+ else:
+ index = faiss.index_cpu_to_all_gpus(index)
+ # Device ids given as a list mean mapping to those devices specified.
+ elif isinstance(device, (list, tuple)):
+ index = faiss.index_cpu_to_gpus_list(index, gpus=list(device))
+ else:
+ raise TypeError(
+ f"The argument type: {type(device)} is not expected. "
+ + "Please pass in either nothing, a positive int, a negative int, or a list of positive ints."
+ )
+
+ return index
+
+ def search(self, query: np.array, k=10, **kwargs) -> SearchResults:
+ """Find the nearest examples indices to the query.
+
+ Args:
+ query (`np.array`): The query as a numpy array.
+ k (`int`): The number of examples to retrieve.
+
+ Output:
+ scores (`List[List[float]`): The retrieval scores of the retrieved examples.
+ indices (`List[List[int]]`): The indices of the retrieved examples.
+ """
+ if len(query.shape) != 1 and (len(query.shape) != 2 or query.shape[0] != 1):
+ raise ValueError("Shape of query is incorrect, it has to be either a 1D array or 2D (1, N)")
+
+ queries = query.reshape(1, -1)
+ if not queries.flags.c_contiguous:
+ queries = np.asarray(queries, order="C")
+ scores, indices = self.faiss_index.search(queries, k, **kwargs)
+ return SearchResults(scores[0], indices[0].astype(int))
+
+ def search_batch(self, queries: np.array, k=10, **kwargs) -> BatchedSearchResults:
+ """Find the nearest examples indices to the queries.
+
+ Args:
+ queries (`np.array`): The queries as a numpy array.
+ k (`int`): The number of examples to retrieve.
+
+ Output:
+ total_scores (`List[List[float]`): The retrieval scores of the retrieved examples per query.
+ total_indices (`List[List[int]]`): The indices of the retrieved examples per query.
+ """
+ if len(queries.shape) != 2:
+ raise ValueError("Shape of query must be 2D")
+ if not queries.flags.c_contiguous:
+ queries = np.asarray(queries, order="C")
+ scores, indices = self.faiss_index.search(queries, k, **kwargs)
+ return BatchedSearchResults(scores, indices.astype(int))
+
+ def save(self, file: Union[str, PurePath], storage_options: Optional[dict] = None):
+ """Serialize the FaissIndex on disk"""
+ import faiss # noqa: F811
+
+ if self.device is not None and isinstance(self.device, (int, list, tuple)):
+ index = faiss.index_gpu_to_cpu(self.faiss_index)
+ else:
+ index = self.faiss_index
+
+ with fsspec.open(str(file), "wb", **(storage_options or {})) as f:
+ faiss.write_index(index, faiss.BufferedIOWriter(faiss.PyCallbackIOWriter(f.write)))
+
+ @classmethod
+ def load(
+ cls,
+ file: Union[str, PurePath],
+ device: Optional[Union[int, list[int]]] = None,
+ storage_options: Optional[dict] = None,
+ ) -> "FaissIndex":
+ """Deserialize the FaissIndex from disk"""
+ import faiss # noqa: F811
+
+ # Instances of FaissIndex is essentially just a wrapper for faiss indices.
+ faiss_index = cls(device=device)
+ with fsspec.open(str(file), "rb", **(storage_options or {})) as f:
+ index = faiss.read_index(faiss.BufferedIOReader(faiss.PyCallbackIOReader(f.read)))
+ faiss_index.faiss_index = faiss_index._faiss_index_to_device(index, faiss_index.device)
+ return faiss_index
+
+
+class IndexableMixin:
+ """Add indexing features to `datasets.Dataset`"""
+
+ def __init__(self):
+ self._indexes: dict[str, BaseIndex] = {}
+
+ def __len__(self):
+ raise NotImplementedError
+
+ def __getitem__(self, key):
+ raise NotImplementedError
+
+ def is_index_initialized(self, index_name: str) -> bool:
+ return index_name in self._indexes
+
+ def _check_index_is_initialized(self, index_name: str):
+ if not self.is_index_initialized(index_name):
+ raise MissingIndex(
+ f"Index with index_name '{index_name}' not initialized yet. Please make sure that you call `add_faiss_index` or `add_elasticsearch_index` first."
+ )
+
+ def list_indexes(self) -> list[str]:
+ """List the `colindex_nameumns`/identifiers of all the attached indexes."""
+ return list(self._indexes)
+
+ def get_index(self, index_name: str) -> BaseIndex:
+ """List the `index_name`/identifiers of all the attached indexes.
+
+ Args:
+ index_name (`str`): Index name.
+
+ Returns:
+ [`BaseIndex`]
+ """
+ self._check_index_is_initialized(index_name)
+ return self._indexes[index_name]
+
+ def add_faiss_index(
+ self,
+ column: str,
+ index_name: Optional[str] = None,
+ device: Optional[Union[int, list[int]]] = None,
+ string_factory: Optional[str] = None,
+ metric_type: Optional[int] = None,
+ custom_index: Optional["faiss.Index"] = None,
+ batch_size: int = 1000,
+ train_size: Optional[int] = None,
+ faiss_verbose: bool = False,
+ ):
+ """Add a dense index using Faiss for fast retrieval.
+ The index is created using the vectors of the specified column.
+ You can specify `device` if you want to run it on GPU (`device` must be the GPU index, see more below).
+ You can find more information about Faiss here:
+ - For `string factory`: https://github.com/facebookresearch/faiss/wiki/The-index-factory
+
+ Args:
+ column (`str`): The column of the vectors to add to the index.
+ index_name (Optional `str`): The index_name/identifier of the index. This is the index_name that is used to call `.get_nearest` or `.search`.
+ By default it corresponds to `column`.
+ device (Optional `Union[int, List[int]]`): If positive integer, this is the index of the GPU to use. If negative integer, use all GPUs.
+ If a list of positive integers is passed in, run only on those GPUs. By default it uses the CPU.
+ string_factory (Optional `str`): This is passed to the index factory of Faiss to create the index. Default index class is IndexFlatIP.
+ metric_type (Optional `int`): Type of metric. Ex: `faiss.METRIC_INNER_PRODUCT` or `faiss.METRIC_L2`.
+ custom_index (Optional `faiss.Index`): Custom Faiss index that you already have instantiated and configured for your needs.
+ batch_size (Optional `int`): Size of the batch to use while adding vectors to the FaissIndex. Default value is 1000.
+
+ train_size (Optional `int`): If the index needs a training step, specifies how many vectors will be used to train the index.
+ faiss_verbose (`bool`, defaults to False): Enable the verbosity of the Faiss index.
+ """
+ index_name = index_name if index_name is not None else column
+ faiss_index = FaissIndex(
+ device=device, string_factory=string_factory, metric_type=metric_type, custom_index=custom_index
+ )
+ faiss_index.add_vectors(
+ self, column=column, batch_size=batch_size, train_size=train_size, faiss_verbose=faiss_verbose
+ )
+ self._indexes[index_name] = faiss_index
+
+ def add_faiss_index_from_external_arrays(
+ self,
+ external_arrays: np.array,
+ index_name: str,
+ device: Optional[Union[int, list[int]]] = None,
+ string_factory: Optional[str] = None,
+ metric_type: Optional[int] = None,
+ custom_index: Optional["faiss.Index"] = None,
+ batch_size: int = 1000,
+ train_size: Optional[int] = None,
+ faiss_verbose: bool = False,
+ ):
+ """Add a dense index using Faiss for fast retrieval.
+ The index is created using the vectors of `external_arrays`.
+ You can specify `device` if you want to run it on GPU (`device` must be the GPU index).
+ You can find more information about Faiss here:
+ - For `string factory`: https://github.com/facebookresearch/faiss/wiki/The-index-factory
+
+ Args:
+ external_arrays (`np.array`): If you want to use arrays from outside the lib for the index, you can set `external_arrays`.
+ It will use `external_arrays` to create the Faiss index instead of the arrays in the given `column`.
+ index_name (`str`): The index_name/identifier of the index. This is the index_name that is used to call `.get_nearest` or `.search`.
+ device (Optional `Union[int, List[int]]`): If positive integer, this is the index of the GPU to use. If negative integer, use all GPUs.
+ If a list of positive integers is passed in, run only on those GPUs. By default it uses the CPU.
+ string_factory (Optional `str`): This is passed to the index factory of Faiss to create the index. Default index class is IndexFlatIP.
+ metric_type (Optional `int`): Type of metric. Ex: `faiss.METRIC_INNER_PRODUCT` or `faiss.METRIC_L2`.
+ custom_index (Optional `faiss.Index`): Custom Faiss index that you already have instantiated and configured for your needs.
+ batch_size (Optional `int`): Size of the batch to use while adding vectors to the FaissIndex. Default value is 1000.
+
+ train_size (Optional `int`): If the index needs a training step, specifies how many vectors will be used to train the index.
+ faiss_verbose (`bool`, defaults to False): Enable the verbosity of the Faiss index.
+ """
+ faiss_index = FaissIndex(
+ device=device, string_factory=string_factory, metric_type=metric_type, custom_index=custom_index
+ )
+ faiss_index.add_vectors(
+ external_arrays, column=None, batch_size=batch_size, train_size=train_size, faiss_verbose=faiss_verbose
+ )
+ self._indexes[index_name] = faiss_index
+
+ def save_faiss_index(self, index_name: str, file: Union[str, PurePath], storage_options: Optional[dict] = None):
+ """Save a FaissIndex on disk.
+
+ Args:
+ index_name (`str`): The index_name/identifier of the index. This is the index_name that is used to call `.get_nearest` or `.search`.
+ file (`str`): The path to the serialized faiss index on disk or remote URI (e.g. `"s3://my-bucket/index.faiss"`).
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+
+
+
+ """
+ index = self.get_index(index_name)
+ if not isinstance(index, FaissIndex):
+ raise ValueError(f"Index '{index_name}' is not a FaissIndex but a '{type(index)}'")
+ index.save(file, storage_options=storage_options)
+ logger.info(f"Saved FaissIndex {index_name} at {file}")
+
+ def load_faiss_index(
+ self,
+ index_name: str,
+ file: Union[str, PurePath],
+ device: Optional[Union[int, list[int]]] = None,
+ storage_options: Optional[dict] = None,
+ ):
+ """Load a FaissIndex from disk.
+
+ If you want to do additional configurations, you can have access to the faiss index object by doing
+ `.get_index(index_name).faiss_index` to make it fit your needs.
+
+ Args:
+ index_name (`str`): The index_name/identifier of the index. This is the index_name that is used to
+ call `.get_nearest` or `.search`.
+ file (`str`): The path to the serialized faiss index on disk or remote URI (e.g. `"s3://my-bucket/index.faiss"`).
+ device (Optional `Union[int, List[int]]`): If positive integer, this is the index of the GPU to use. If negative integer, use all GPUs.
+ If a list of positive integers is passed in, run only on those GPUs. By default it uses the CPU.
+ storage_options (`dict`, *optional*):
+ Key/value pairs to be passed on to the file-system backend, if any.
+
+
+
+ """
+ index = FaissIndex.load(file, device=device, storage_options=storage_options)
+ if index.faiss_index.ntotal != len(self):
+ raise ValueError(
+ f"Index size should match Dataset size, but Index '{index_name}' at {file} has {index.faiss_index.ntotal} elements while the dataset has {len(self)} examples."
+ )
+ self._indexes[index_name] = index
+ logger.info(f"Loaded FaissIndex {index_name} from {file}")
+
+ def add_elasticsearch_index(
+ self,
+ column: str,
+ index_name: Optional[str] = None,
+ host: Optional[str] = None,
+ port: Optional[int] = None,
+ es_client: Optional["Elasticsearch"] = None,
+ es_index_name: Optional[str] = None,
+ es_index_config: Optional[dict] = None,
+ ):
+ """Add a text index using ElasticSearch for fast retrieval.
+
+ Args:
+ column (`str`): The column of the documents to add to the index.
+ index_name (Optional `str`): The index_name/identifier of the index. This is the index name that is used to call `.get_nearest` or `.search`.
+ By default it corresponds to `column`.
+ host (Optional `str`, defaults to localhost):
+ host of where ElasticSearch is running
+ port (Optional `str`, defaults to 9200):
+ port of where ElasticSearch is running
+ es_client (Optional `elasticsearch.Elasticsearch`):
+ The elasticsearch client used to create the index if host and port are None.
+ es_index_name (Optional `str`): The elasticsearch index name used to create the index.
+ es_index_config (Optional `dict`):
+ The configuration of the elasticsearch index.
+ Default config is:
+
+ Config::
+
+ {
+ "settings": {
+ "number_of_shards": 1,
+ "analysis": {"analyzer": {"stop_standard": {"type": "standard", " stopwords": "_english_"}}},
+ },
+ "mappings": {
+ "properties": {
+ "text": {
+ "type": "text",
+ "analyzer": "standard",
+ "similarity": "BM25"
+ },
+ }
+ },
+ }
+ """
+ index_name = index_name if index_name is not None else column
+ es_index = ElasticSearchIndex(
+ host=host, port=port, es_client=es_client, es_index_name=es_index_name, es_index_config=es_index_config
+ )
+ es_index.add_documents(self, column=column)
+ self._indexes[index_name] = es_index
+
+ def load_elasticsearch_index(
+ self,
+ index_name: str,
+ es_index_name: str,
+ host: Optional[str] = None,
+ port: Optional[int] = None,
+ es_client: Optional["Elasticsearch"] = None,
+ es_index_config: Optional[dict] = None,
+ ):
+ """Load an existing text index using ElasticSearch for fast retrieval.
+
+ Args:
+ index_name (`str`):
+ The `index_name`/identifier of the index. This is the index name that is used to call `get_nearest` or `search`.
+ es_index_name (`str`):
+ The name of elasticsearch index to load.
+ host (`str`, *optional*, defaults to `localhost`):
+ Host of where ElasticSearch is running.
+ port (`str`, *optional*, defaults to `9200`):
+ Port of where ElasticSearch is running.
+ es_client (`elasticsearch.Elasticsearch`, *optional*):
+ The elasticsearch client used to create the index if host and port are `None`.
+ es_index_config (`dict`, *optional*):
+ The configuration of the elasticsearch index.
+ Default config is:
+ ```
+ {
+ "settings": {
+ "number_of_shards": 1,
+ "analysis": {"analyzer": {"stop_standard": {"type": "standard", " stopwords": "_english_"}}},
+ },
+ "mappings": {
+ "properties": {
+ "text": {
+ "type": "text",
+ "analyzer": "standard",
+ "similarity": "BM25"
+ },
+ }
+ },
+ }
+ ```
+ """
+ self._indexes[index_name] = ElasticSearchIndex(
+ host=host, port=port, es_client=es_client, es_index_name=es_index_name, es_index_config=es_index_config
+ )
+
+ def drop_index(self, index_name: str):
+ """Drop the index with the specified column.
+
+ Args:
+ index_name (`str`):
+ The `index_name`/identifier of the index.
+ """
+ del self._indexes[index_name]
+
+ def search(self, index_name: str, query: Union[str, np.array], k: int = 10, **kwargs) -> SearchResults:
+ """Find the nearest examples indices in the dataset to the query.
+
+ Args:
+ index_name (`str`):
+ The name/identifier of the index.
+ query (`Union[str, np.ndarray]`):
+ The query as a string if `index_name` is a text index or as a numpy array if `index_name` is a vector index.
+ k (`int`):
+ The number of examples to retrieve.
+
+ Returns:
+ `(scores, indices)`:
+ A tuple of `(scores, indices)` where:
+ - **scores** (`List[List[float]`): the retrieval scores from either FAISS (`IndexFlatL2` by default) or ElasticSearch of the retrieved examples
+ - **indices** (`List[List[int]]`): the indices of the retrieved examples
+ """
+ self._check_index_is_initialized(index_name)
+ return self._indexes[index_name].search(query, k, **kwargs)
+
+ def search_batch(
+ self, index_name: str, queries: Union[list[str], np.array], k: int = 10, **kwargs
+ ) -> BatchedSearchResults:
+ """Find the nearest examples indices in the dataset to the query.
+
+ Args:
+ index_name (`str`):
+ The `index_name`/identifier of the index.
+ queries (`Union[List[str], np.ndarray]`):
+ The queries as a list of strings if `index_name` is a text index or as a numpy array if `index_name` is a vector index.
+ k (`int`):
+ The number of examples to retrieve per query.
+
+ Returns:
+ `(total_scores, total_indices)`:
+ A tuple of `(total_scores, total_indices)` where:
+ - **total_scores** (`List[List[float]`): the retrieval scores from either FAISS (`IndexFlatL2` by default) or ElasticSearch of the retrieved examples per query
+ - **total_indices** (`List[List[int]]`): the indices of the retrieved examples per query
+ """
+ self._check_index_is_initialized(index_name)
+ return self._indexes[index_name].search_batch(queries, k, **kwargs)
+
+ def get_nearest_examples(
+ self, index_name: str, query: Union[str, np.array], k: int = 10, **kwargs
+ ) -> NearestExamplesResults:
+ """Find the nearest examples in the dataset to the query.
+
+ Args:
+ index_name (`str`):
+ The index_name/identifier of the index.
+ query (`Union[str, np.ndarray]`):
+ The query as a string if `index_name` is a text index or as a numpy array if `index_name` is a vector index.
+ k (`int`):
+ The number of examples to retrieve.
+
+ Returns:
+ `(scores, examples)`:
+ A tuple of `(scores, examples)` where:
+ - **scores** (`List[float]`): the retrieval scores from either FAISS (`IndexFlatL2` by default) or ElasticSearch of the retrieved examples
+ - **examples** (`dict`): the retrieved examples
+ """
+ self._check_index_is_initialized(index_name)
+ scores, indices = self.search(index_name, query, k, **kwargs)
+ top_indices = [i for i in indices if i >= 0]
+ return NearestExamplesResults(scores[: len(top_indices)], self[top_indices])
+
+ def get_nearest_examples_batch(
+ self, index_name: str, queries: Union[list[str], np.array], k: int = 10, **kwargs
+ ) -> BatchedNearestExamplesResults:
+ """Find the nearest examples in the dataset to the query.
+
+ Args:
+ index_name (`str`):
+ The `index_name`/identifier of the index.
+ queries (`Union[List[str], np.ndarray]`):
+ The queries as a list of strings if `index_name` is a text index or as a numpy array if `index_name` is a vector index.
+ k (`int`):
+ The number of examples to retrieve per query.
+
+ Returns:
+ `(total_scores, total_examples)`:
+ A tuple of `(total_scores, total_examples)` where:
+ - **total_scores** (`List[List[float]`): the retrieval scores from either FAISS (`IndexFlatL2` by default) or ElasticSearch of the retrieved examples per query
+ - **total_examples** (`List[dict]`): the retrieved examples per query
+ """
+ self._check_index_is_initialized(index_name)
+ total_scores, total_indices = self.search_batch(index_name, queries, k, **kwargs)
+ total_scores = [
+ scores_i[: len([i for i in indices_i if i >= 0])]
+ for scores_i, indices_i in zip(total_scores, total_indices)
+ ]
+ total_samples = [self[[i for i in indices if i >= 0]] for indices in total_indices]
+ return BatchedNearestExamplesResults(total_scores, total_samples)
diff --git a/venv/Lib/site-packages/datasets/splits.py b/venv/Lib/site-packages/datasets/splits.py
new file mode 100644
index 0000000000000000000000000000000000000000..7e8ea953afdbb76240bafa272a33b27ef237359d
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/splits.py
@@ -0,0 +1,647 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Splits related API."""
+
+import abc
+import collections
+import copy
+import dataclasses
+import re
+from dataclasses import dataclass
+from typing import Optional, Union
+
+from .arrow_reader import FileInstructions, make_file_instructions
+from .naming import _split_re
+from .utils.py_utils import NonMutableDict, asdict
+
+
+@dataclass
+class SplitInfo:
+ name: str = dataclasses.field(default="", metadata={"include_in_asdict_even_if_is_default": True})
+ num_bytes: int = dataclasses.field(default=0, metadata={"include_in_asdict_even_if_is_default": True})
+ num_examples: int = dataclasses.field(default=0, metadata={"include_in_asdict_even_if_is_default": True})
+ shard_lengths: Optional[list[int]] = None
+ original_shard_lengths: Optional[list[int]] = None
+
+ # Deprecated
+ # For backward compatibility, this field needs to always be included in files like
+ # dataset_infos.json and dataset_info.json files
+ # To do so, we always include it in the output of datasets.utils.py_utils.asdict(split_info)
+ dataset_name: Optional[str] = dataclasses.field(
+ default=None, metadata={"include_in_asdict_even_if_is_default": True}
+ )
+
+ @property
+ def file_instructions(self):
+ """Returns the list of dict(filename, take, skip)."""
+ # `self.dataset_name` is assigned in `SplitDict.add()`.
+ instructions = make_file_instructions(
+ name=self.dataset_name,
+ split_infos=[self],
+ instruction=str(self.name),
+ )
+ return instructions.file_instructions
+
+ def __repr__(self):
+ return (
+ self.__class__.__qualname__
+ + "("
+ + ", ".join(
+ [f"{f.name}={repr(getattr(self, f.name))}" for f in dataclasses.fields(self) if getattr(self, f.name)]
+ )
+ + ")"
+ )
+
+
+@dataclass
+class SubSplitInfo:
+ """Wrapper around a sub split info.
+ This class exposes info on the subsplit:
+ ```
+ ds, info = datasets.load_dataset(..., split='train[75%:]', with_info=True)
+ info.splits['train[75%:]'].num_examples
+ ```
+ """
+
+ instructions: FileInstructions
+
+ @property
+ def num_examples(self):
+ """Returns the number of examples in the subsplit."""
+ return self.instructions.num_examples
+
+ @property
+ def file_instructions(self):
+ """Returns the list of dict(filename, take, skip)."""
+ return self.instructions.file_instructions
+
+
+class SplitBase(metaclass=abc.ABCMeta):
+ # pylint: disable=line-too-long
+ """Abstract base class for Split compositionality.
+
+ See the
+ [guide on splits](../loading#slice-splits)
+ for more information.
+
+ There are three parts to the composition:
+ 1) The splits are composed (defined, merged, split,...) together before
+ calling the `.as_dataset()` function. This is done with the `__add__`,
+ `__getitem__`, which return a tree of `SplitBase` (whose leaf
+ are the `NamedSplit` objects)
+
+ ```
+ split = datasets.Split.TRAIN + datasets.Split.TEST.subsplit(datasets.percent[:50])
+ ```
+
+ 2) The `SplitBase` is forwarded to the `.as_dataset()` function
+ to be resolved into actual read instruction. This is done by the
+ `.get_read_instruction()` method which takes the real dataset splits
+ (name, number of shards,...) and parse the tree to return a
+ `SplitReadInstruction()` object
+
+ ```
+ read_instruction = split.get_read_instruction(self.info.splits)
+ ```
+
+ 3) The `SplitReadInstruction` is then used in the `tf.data.Dataset` pipeline
+ to define which files to read and how to skip examples within file.
+
+ """
+
+ # pylint: enable=line-too-long
+
+ @abc.abstractmethod
+ def get_read_instruction(self, split_dict):
+ """Parse the descriptor tree and compile all read instructions together.
+
+ Args:
+ split_dict: `dict`, The `dict[split_name, SplitInfo]` of the dataset
+
+ Returns:
+ split_read_instruction: `SplitReadInstruction`
+ """
+ raise NotImplementedError("Abstract method")
+
+ def __eq__(self, other):
+ """Equality: datasets.Split.TRAIN == 'train'."""
+ if isinstance(other, (NamedSplit, str)):
+ return False
+ raise NotImplementedError("Equality is not implemented between merged/sub splits.")
+
+ def __ne__(self, other):
+ """InEquality: datasets.Split.TRAIN != 'test'."""
+ return not self.__eq__(other)
+
+ def __add__(self, other):
+ """Merging: datasets.Split.TRAIN + datasets.Split.TEST."""
+ return _SplitMerged(self, other)
+
+ def subsplit(self, arg=None, k=None, percent=None, weighted=None): # pylint: disable=redefined-outer-name
+ """Divides this split into subsplits.
+
+ There are 3 ways to define subsplits, which correspond to the 3
+ arguments `k` (get `k` even subsplits), `percent` (get a slice of the
+ dataset with `datasets.percent`), and `weighted` (get subsplits with proportions
+ specified by `weighted`).
+
+ Example::
+
+ ```
+ # 50% train, 50% test
+ train, test = split.subsplit(k=2)
+ # 50% train, 25% test, 25% validation
+ train, test, validation = split.subsplit(weighted=[2, 1, 1])
+ # Extract last 20%
+ subsplit = split.subsplit(datasets.percent[-20:])
+ ```
+
+ Warning: k and weighted will be converted into percent which mean that
+ values below the percent will be rounded up or down. The final split may be
+ bigger to deal with remainders. For instance:
+
+ ```
+ train, test, valid = split.subsplit(k=3) # 33%, 33%, 34%
+ s1, s2, s3, s4 = split.subsplit(weighted=[2, 2, 1, 1]) # 33%, 33%, 16%, 18%
+ ```
+
+ Args:
+ arg: If no kwargs are given, `arg` will be interpreted as one of
+ `k`, `percent`, or `weighted` depending on the type.
+ For example:
+ ```
+ split.subsplit(10) # Equivalent to split.subsplit(k=10)
+ split.subsplit(datasets.percent[:-20]) # percent=datasets.percent[:-20]
+ split.subsplit([1, 1, 2]) # weighted=[1, 1, 2]
+ ```
+ k: `int` If set, subdivide the split into `k` equal parts.
+ percent: `datasets.percent slice`, return a single subsplit corresponding to
+ a slice of the original split. For example:
+ `split.subsplit(datasets.percent[-20:]) # Last 20% of the dataset`.
+ weighted: `list[int]`, return a list of subsplits whose proportions match
+ the normalized sum of the list. For example:
+ `split.subsplit(weighted=[1, 1, 2]) # 25%, 25%, 50%`.
+
+ Returns:
+ A subsplit or list of subsplits extracted from this split object.
+ """
+ # Note that the percent kwargs redefine the outer name datasets.percent. This
+ # is done for consistency (.subsplit(percent=datasets.percent[:40]))
+ if sum(bool(x) for x in (arg, k, percent, weighted)) != 1:
+ raise ValueError("Only one argument of subsplit should be set.")
+
+ # Auto deduce k
+ if isinstance(arg, int):
+ k = arg
+ elif isinstance(arg, slice):
+ percent = arg
+ elif isinstance(arg, list):
+ weighted = arg
+
+ if not (k or percent or weighted):
+ raise ValueError(
+ f"Invalid split argument {arg}. Only list, slice and int supported. "
+ "One of k, weighted or percent should be set to a non empty value."
+ )
+
+ def assert_slices_coverage(slices):
+ # Ensure that the expended slices cover all percents.
+ assert sum((list(range(*s.indices(100))) for s in slices), []) == list(range(100))
+
+ if k:
+ if not 0 < k <= 100:
+ raise ValueError(f"Subsplit k should be between 0 and 100, got {k}")
+ shift = 100 // k
+ slices = [slice(i * shift, (i + 1) * shift) for i in range(k)]
+ # Round up last element to ensure all elements are taken
+ slices[-1] = slice(slices[-1].start, 100)
+ # Internal check to ensure full coverage
+ assert_slices_coverage(slices)
+ return tuple(_SubSplit(self, s) for s in slices)
+ elif percent:
+ return _SubSplit(self, percent)
+ elif weighted:
+ # Normalize the weighted sum
+ total = sum(weighted)
+ weighted = [100 * x // total for x in weighted]
+ # Create the slice for each of the elements
+ start = 0
+ stop = 0
+ slices = []
+ for v in weighted:
+ stop += v
+ slices.append(slice(start, stop))
+ start = stop
+ # Round up last element to ensure all elements are taken
+ slices[-1] = slice(slices[-1].start, 100)
+ # Internal check to ensure full coverage
+ assert_slices_coverage(slices)
+ return tuple(_SubSplit(self, s) for s in slices)
+ else:
+ # Should not be possible
+ raise ValueError("Could not determine the split")
+
+
+# 2 requirements:
+# 1. datasets.percent be sliceable
+# 2. datasets.percent be documented
+#
+# Instances are not documented, so we want datasets.percent to be a class, but to
+# have it be sliceable, we need this metaclass.
+class PercentSliceMeta(type):
+ def __getitem__(cls, slice_value):
+ if not isinstance(slice_value, slice):
+ raise ValueError(f"datasets.percent should only be called with slice, not {slice_value}")
+ return slice_value
+
+
+class PercentSlice(metaclass=PercentSliceMeta):
+ # pylint: disable=line-too-long
+ """Syntactic sugar for defining slice subsplits: `datasets.percent[75:-5]`.
+
+ See the
+ [guide on splits](../loading#slice-splits)
+ for more information.
+ """
+
+ # pylint: enable=line-too-long
+ pass
+
+
+percent = PercentSlice # pylint: disable=invalid-name
+
+
+class _SplitMerged(SplitBase):
+ """Represent two split descriptors merged together."""
+
+ def __init__(self, split1, split2):
+ self._split1 = split1
+ self._split2 = split2
+
+ def get_read_instruction(self, split_dict):
+ read_instruction1 = self._split1.get_read_instruction(split_dict)
+ read_instruction2 = self._split2.get_read_instruction(split_dict)
+ return read_instruction1 + read_instruction2
+
+ def __repr__(self):
+ return f"({repr(self._split1)} + {repr(self._split2)})"
+
+
+class _SubSplit(SplitBase):
+ """Represent a sub split of a split descriptor."""
+
+ def __init__(self, split, slice_value):
+ self._split = split
+ self._slice_value = slice_value
+
+ def get_read_instruction(self, split_dict):
+ return self._split.get_read_instruction(split_dict)[self._slice_value]
+
+ def __repr__(self):
+ slice_str = "{start}:{stop}"
+ if self._slice_value.step is not None:
+ slice_str += ":{step}"
+ slice_str = slice_str.format(
+ start="" if self._slice_value.start is None else self._slice_value.start,
+ stop="" if self._slice_value.stop is None else self._slice_value.stop,
+ step=self._slice_value.step,
+ )
+ return f"{repr(self._split)}(datasets.percent[{slice_str}])"
+
+
+class NamedSplit(SplitBase):
+ """Descriptor corresponding to a named split (train, test, ...).
+
+ Example:
+ Each descriptor can be composed with other using addition or slice:
+
+ ```py
+ split = datasets.Split.TRAIN.subsplit(datasets.percent[0:25]) + datasets.Split.TEST
+ ```
+
+ The resulting split will correspond to 25% of the train split merged with
+ 100% of the test split.
+
+ A split cannot be added twice, so the following will fail:
+
+ ```py
+ split = (
+ datasets.Split.TRAIN.subsplit(datasets.percent[:25]) +
+ datasets.Split.TRAIN.subsplit(datasets.percent[75:])
+ ) # Error
+ split = datasets.Split.TEST + datasets.Split.ALL # Error
+ ```
+
+ The slices can be applied only one time. So the following are valid:
+
+ ```py
+ split = (
+ datasets.Split.TRAIN.subsplit(datasets.percent[:25]) +
+ datasets.Split.TEST.subsplit(datasets.percent[:50])
+ )
+ split = (datasets.Split.TRAIN + datasets.Split.TEST).subsplit(datasets.percent[:50])
+ ```
+
+ But this is not valid:
+
+ ```py
+ train = datasets.Split.TRAIN
+ test = datasets.Split.TEST
+ split = train.subsplit(datasets.percent[:25]).subsplit(datasets.percent[:25])
+ split = (train.subsplit(datasets.percent[:25]) + test).subsplit(datasets.percent[:50])
+ ```
+ """
+
+ def __init__(self, name: str):
+ self._name = name
+ split_names_from_instruction = [split_instruction.split("[")[0] for split_instruction in name.split("+")]
+ for split_name in split_names_from_instruction:
+ if not re.match(_split_re, split_name):
+ raise ValueError(f"Split name should match '{_split_re}' but got '{split_name}'.")
+
+ def __str__(self):
+ return self._name
+
+ def __repr__(self):
+ return f"NamedSplit({self._name!r})"
+
+ def __eq__(self, other):
+ """Equality: datasets.Split.TRAIN == 'train'."""
+ if isinstance(other, NamedSplit):
+ return self._name == other._name # pylint: disable=protected-access
+ elif isinstance(other, SplitBase):
+ return False
+ elif isinstance(other, str): # Other should be string
+ return self._name == other
+ else:
+ return False
+
+ def __lt__(self, other):
+ return self._name < other._name # pylint: disable=protected-access
+
+ def __hash__(self):
+ return hash(self._name)
+
+ def get_read_instruction(self, split_dict):
+ return SplitReadInstruction(split_dict[self._name])
+
+
+class NamedSplitAll(NamedSplit):
+ """Split corresponding to the union of all defined dataset splits."""
+
+ def __init__(self):
+ super().__init__("all")
+
+ def __repr__(self):
+ return "NamedSplitAll()"
+
+ def get_read_instruction(self, split_dict):
+ # Merge all dataset splits together
+ read_instructions = [SplitReadInstruction(s) for s in split_dict.values()]
+ return sum(read_instructions, SplitReadInstruction())
+
+
+class Split:
+ # pylint: disable=line-too-long
+ """`Enum` for dataset splits.
+
+ Datasets are typically split into different subsets to be used at various
+ stages of training and evaluation.
+
+ - `TRAIN`: the training data.
+ - `VALIDATION`: the validation data. If present, this is typically used as
+ evaluation data while iterating on a model (e.g. changing hyperparameters,
+ model architecture, etc.).
+ - `TEST`: the testing data. This is the data to report metrics on. Typically
+ you do not want to use this during model iteration as you may overfit to it.
+ - `ALL`: the union of all defined dataset splits.
+
+ All splits, including compositions inherit from `datasets.SplitBase`.
+
+ See the [guide](../load_hub#splits) on splits for more information.
+
+ Example:
+
+ ```py
+ >>> datasets.SplitGenerator(
+ ... name=datasets.Split.TRAIN,
+ ... gen_kwargs={"split_key": "train", "files": dl_manager.download_and extract(url)},
+ ... ),
+ ... datasets.SplitGenerator(
+ ... name=datasets.Split.VALIDATION,
+ ... gen_kwargs={"split_key": "validation", "files": dl_manager.download_and extract(url)},
+ ... ),
+ ... datasets.SplitGenerator(
+ ... name=datasets.Split.TEST,
+ ... gen_kwargs={"split_key": "test", "files": dl_manager.download_and extract(url)},
+ ... )
+ ```
+ """
+
+ # pylint: enable=line-too-long
+ TRAIN = NamedSplit("train")
+ TEST = NamedSplit("test")
+ VALIDATION = NamedSplit("validation")
+ ALL = NamedSplitAll()
+
+ def __new__(cls, name):
+ """Create a custom split with datasets.Split('custom_name')."""
+ return NamedSplitAll() if name == "all" else NamedSplit(name)
+
+
+# Similar to SplitInfo, but contain an additional slice info
+SlicedSplitInfo = collections.namedtuple(
+ "SlicedSplitInfo",
+ [
+ "split_info",
+ "slice_value",
+ ],
+) # noqa: E231
+
+
+class SplitReadInstruction:
+ """Object containing the reading instruction for the dataset.
+
+ Similarly to `SplitDescriptor` nodes, this object can be composed with itself,
+ but the resolution happens instantaneously, instead of keeping track of the
+ tree, such as all instructions are compiled and flattened in a single
+ SplitReadInstruction object containing the list of files and slice to use.
+
+ Once resolved, the instructions can be accessed with:
+
+ ```
+ read_instructions.get_list_sliced_split_info() # List of splits to use
+ ```
+
+ """
+
+ def __init__(self, split_info=None):
+ self._splits = NonMutableDict(error_msg="Overlap between splits. Split {key} has been added with itself.")
+
+ if split_info:
+ self.add(SlicedSplitInfo(split_info=split_info, slice_value=None))
+
+ def add(self, sliced_split):
+ """Add a SlicedSplitInfo the read instructions."""
+ # TODO(epot): Check that the number of examples per shard % 100 == 0
+ # Otherwise the slices value may be unbalanced and not exactly reflect the
+ # requested slice.
+ self._splits[sliced_split.split_info.name] = sliced_split
+
+ def __add__(self, other):
+ """Merging split together."""
+ # Will raise error if a split has already be added (NonMutableDict)
+ # TODO(epot): If a split is already added but there is no overlap between
+ # the slices, should merge the slices (ex: [:10] + [80:])
+ split_instruction = SplitReadInstruction()
+ split_instruction._splits.update(self._splits) # pylint: disable=protected-access
+ split_instruction._splits.update(other._splits) # pylint: disable=protected-access
+ return split_instruction
+
+ def __getitem__(self, slice_value):
+ """Sub-splits."""
+ # Will raise an error if a split has already been sliced
+ split_instruction = SplitReadInstruction()
+ for v in self._splits.values():
+ if v.slice_value is not None:
+ raise ValueError(f"Trying to slice Split {v.split_info.name} which has already been sliced")
+ v = v._asdict()
+ v["slice_value"] = slice_value
+ split_instruction.add(SlicedSplitInfo(**v))
+ return split_instruction
+
+ def get_list_sliced_split_info(self):
+ return list(self._splits.values())
+
+
+class SplitDict(dict[str, SplitInfo]):
+ """Split info object."""
+
+ def __init__(self, *args, dataset_name=None, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.dataset_name = dataset_name
+
+ def __getitem__(self, key: Union[SplitBase, str]):
+ # 1st case: The key exists: `info.splits['train']`
+ if str(key) in self:
+ return super().__getitem__(str(key))
+ # 2nd case: Uses instructions: `info.splits['train[50%]']`
+ else:
+ instructions = make_file_instructions(
+ name=self.dataset_name,
+ split_infos=self.values(),
+ instruction=key,
+ )
+ return SubSplitInfo(instructions)
+
+ def __setitem__(self, key: Union[SplitBase, str], value: SplitInfo):
+ if key != value.name:
+ raise ValueError(f"Cannot add elem. (key mismatch: '{key}' != '{value.name}')")
+ super().__setitem__(key, value)
+
+ def add(self, split_info: SplitInfo):
+ """Add the split info."""
+ if split_info.name in self:
+ raise ValueError(f"Split {split_info.name} already present")
+ split_info.dataset_name = self.dataset_name
+ super().__setitem__(split_info.name, split_info)
+
+ @property
+ def total_num_examples(self):
+ """Return the total number of examples."""
+ return sum(s.num_examples for s in self.values())
+
+ @classmethod
+ def from_split_dict(cls, split_infos: Union[list, dict], dataset_name: Optional[str] = None):
+ """Returns a new SplitDict initialized from a Dict or List of `split_infos`."""
+ if isinstance(split_infos, dict):
+ split_infos = list(split_infos.values())
+
+ if dataset_name is None:
+ dataset_name = split_infos[0].get("dataset_name") if split_infos else None
+
+ split_dict = cls(dataset_name=dataset_name)
+
+ for split_info in split_infos:
+ if isinstance(split_info, dict):
+ split_info = SplitInfo(**split_info)
+ split_dict.add(split_info)
+
+ return split_dict
+
+ def to_split_dict(self):
+ """Returns a list of SplitInfo protos that we have."""
+ out = []
+ for split_name, split_info in self.items():
+ split_info = copy.deepcopy(split_info)
+ split_info.name = split_name
+ out.append(split_info)
+ return out
+
+ def copy(self):
+ return SplitDict.from_split_dict(self.to_split_dict(), self.dataset_name)
+
+ def _to_yaml_list(self) -> list:
+ out = [asdict(s) for s in self.to_split_dict()]
+ # we don't need the shard lengths in YAML
+ for split_info_dict in out:
+ split_info_dict.pop("shard_lengths", None)
+ split_info_dict.pop("original_shard_lengths", None)
+ # we don't need the dataset_name attribute that is deprecated
+ for split_info_dict in out:
+ split_info_dict.pop("dataset_name", None)
+ return out
+
+ @classmethod
+ def _from_yaml_list(cls, yaml_data: list) -> "SplitDict":
+ return cls.from_split_dict(yaml_data)
+
+
+@dataclass
+class SplitGenerator:
+ """Defines the split information for the generator.
+
+ This should be used as returned value of
+ `GeneratorBasedBuilder._split_generators`.
+ See `GeneratorBasedBuilder._split_generators` for more info and example
+ of usage.
+
+ Args:
+ name (`str`):
+ Name of the `Split` for which the generator will
+ create the examples.
+ **gen_kwargs (additional keyword arguments):
+ Keyword arguments to forward to the `DatasetBuilder._generate_examples` method
+ of the builder.
+
+ Example:
+
+ ```py
+ >>> datasets.SplitGenerator(
+ ... name=datasets.Split.TRAIN,
+ ... gen_kwargs={"split_key": "train", "files": dl_manager.download_and_extract(url)},
+ ... )
+ ```
+ """
+
+ name: str
+ gen_kwargs: dict = dataclasses.field(default_factory=dict)
+ split_info: SplitInfo = dataclasses.field(init=False)
+
+ def __post_init__(self):
+ self.name = str(self.name) # Make sure we convert NamedSplits in strings
+ NamedSplit(self.name) # check that it's a valid split name
+ self.split_info = SplitInfo(name=self.name)
diff --git a/venv/Lib/site-packages/datasets/streaming.py b/venv/Lib/site-packages/datasets/streaming.py
new file mode 100644
index 0000000000000000000000000000000000000000..6e5e8ee15f9f529d93d43dc003ccdef3436fd725
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/streaming.py
@@ -0,0 +1,131 @@
+import importlib
+from functools import wraps
+from typing import TYPE_CHECKING, Optional
+
+from .download.download_config import DownloadConfig
+from .utils.file_utils import (
+ xbasename,
+ xdirname,
+ xet_parse,
+ xexists,
+ xgetsize,
+ xglob,
+ xgzip_open,
+ xisdir,
+ xisfile,
+ xjoin,
+ xlistdir,
+ xnumpy_load,
+ xopen,
+ xpandas_read_csv,
+ xpandas_read_excel,
+ xPath,
+ xpyarrow_parquet_read_table,
+ xrelpath,
+ xsio_loadmat,
+ xsplit,
+ xsplitext,
+ xwalk,
+ xxml_dom_minidom_parse,
+)
+from .utils.logging import get_logger
+from .utils.patching import patch_submodule
+
+
+logger = get_logger(__name__)
+
+
+if TYPE_CHECKING:
+ from .builder import DatasetBuilder
+
+
+def extend_module_for_streaming(module_path, download_config: Optional[DownloadConfig] = None):
+ """Extend the module to support streaming.
+
+ We patch some functions in the module to use `fsspec` to support data streaming:
+ - We use `fsspec.open` to open and read remote files. We patch the module function:
+ - `open`
+ - We use the "::" hop separator to join paths and navigate remote compressed/archive files. We patch the module
+ functions:
+ - `os.path.join`
+ - `pathlib.Path.joinpath` and `pathlib.Path.__truediv__` (called when using the "/" operator)
+
+ The patched functions are replaced with custom functions defined to work with the
+ :class:`~download.streaming_download_manager.StreamingDownloadManager`.
+
+ Args:
+ module_path: Path to the module to be extended.
+ download_config: Mainly use `token` or `storage_options` to support different platforms and auth types.
+ """
+
+ module = importlib.import_module(module_path)
+
+ # TODO(QL): always update the module to add subsequent new authentication without removing old ones
+ if hasattr(module, "_patched_for_streaming") and module._patched_for_streaming:
+ if isinstance(module._patched_for_streaming, DownloadConfig):
+ module._patched_for_streaming.token = download_config.token
+ module._patched_for_streaming.storage_options = download_config.storage_options
+ return
+
+ def wrap_auth(function):
+ @wraps(function)
+ def wrapper(*args, **kwargs):
+ return function(*args, download_config=download_config, **kwargs)
+
+ wrapper._decorator_name_ = "wrap_auth"
+ return wrapper
+
+ # open files in a streaming fashion
+ patch_submodule(module, "open", wrap_auth(xopen)).start()
+ patch_submodule(module, "os.listdir", wrap_auth(xlistdir)).start()
+ patch_submodule(module, "os.walk", wrap_auth(xwalk)).start()
+ patch_submodule(module, "glob.glob", wrap_auth(xglob)).start()
+ # allow to navigate in remote zip files
+ patch_submodule(module, "os.path.join", xjoin).start()
+ patch_submodule(module, "os.path.dirname", xdirname).start()
+ patch_submodule(module, "os.path.basename", xbasename).start()
+ patch_submodule(module, "os.path.relpath", xrelpath).start()
+ patch_submodule(module, "os.path.split", xsplit).start()
+ patch_submodule(module, "os.path.splitext", xsplitext).start()
+ # allow checks on paths
+ patch_submodule(module, "os.path.exists", wrap_auth(xexists)).start()
+ patch_submodule(module, "os.path.isdir", wrap_auth(xisdir)).start()
+ patch_submodule(module, "os.path.isfile", wrap_auth(xisfile)).start()
+ patch_submodule(module, "os.path.getsize", wrap_auth(xgetsize)).start()
+ patch_submodule(module, "pathlib.Path", xPath).start()
+ # file readers
+ patch_submodule(module, "gzip.open", wrap_auth(xgzip_open)).start()
+ patch_submodule(module, "numpy.load", wrap_auth(xnumpy_load)).start()
+ patch_submodule(module, "pandas.read_csv", wrap_auth(xpandas_read_csv), attrs=["__version__"]).start()
+ patch_submodule(module, "pandas.read_excel", wrap_auth(xpandas_read_excel), attrs=["__version__"]).start()
+ patch_submodule(module, "scipy.io.loadmat", wrap_auth(xsio_loadmat), attrs=["__version__"]).start()
+ patch_submodule(module, "xml.etree.ElementTree.parse", wrap_auth(xet_parse)).start()
+ patch_submodule(module, "xml.dom.minidom.parse", wrap_auth(xxml_dom_minidom_parse)).start()
+ # pyarrow: do not patch pyarrow attribute in packaged modules
+ if not module.__name__.startswith("datasets.packaged_modules."):
+ patch_submodule(module, "pyarrow.parquet.read_table", wrap_auth(xpyarrow_parquet_read_table)).start()
+ module._patched_for_streaming = download_config
+
+
+def extend_dataset_builder_for_streaming(builder: "DatasetBuilder"):
+ """Extend the dataset builder module and the modules imported by it to support streaming.
+
+ Args:
+ builder (:class:`DatasetBuilder`): Dataset builder instance.
+ """
+ # this extends the open and os.path.join functions for data streaming
+ download_config = DownloadConfig(storage_options=builder.storage_options, token=builder.token)
+ extend_module_for_streaming(builder.__module__, download_config=download_config)
+
+ # builders can inherit from other builders that might use streaming functionality
+ # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation)
+ # but these parents builders are not patched automatically as they are not instantiated, so we patch them here
+ from .builder import DatasetBuilder
+
+ parent_builder_modules = [
+ cls.__module__
+ for cls in type(builder).__mro__[1:] # make sure it's not the same module we've already patched
+ if issubclass(cls, DatasetBuilder) and cls.__module__ != DatasetBuilder.__module__
+ ] # check it's not a standard builder from datasets.builder
+ for module in parent_builder_modules:
+ extend_module_for_streaming(module, download_config=download_config)
diff --git a/venv/Lib/site-packages/datasets/table.py b/venv/Lib/site-packages/datasets/table.py
new file mode 100644
index 0000000000000000000000000000000000000000..31aeeded7a121a9d30e015e751809229790c4c39
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/table.py
@@ -0,0 +1,2482 @@
+import copy
+import os
+from collections.abc import Iterator
+from functools import partial
+from itertools import groupby
+from typing import TYPE_CHECKING, Any, Callable, Optional, TypeVar, Union
+
+import numpy as np
+import pyarrow as pa
+import pyarrow.compute as pc
+
+from .utils.logging import get_logger
+
+
+if TYPE_CHECKING:
+ from .features.features import Features, FeatureType
+
+
+logger = get_logger(__name__)
+
+
+def inject_arrow_table_documentation(arrow_table_method):
+ def wrapper(fn):
+ fn.__doc__ = arrow_table_method.__doc__ + (fn.__doc__ if fn.__doc__ is not None else "")
+ fn.__doc__ = fn.__doc__.replace("pyarrow.Table", "Table")
+ if hasattr(arrow_table_method, "__annotations__"):
+ fn.__annotations__ = arrow_table_method.__annotations__
+ return fn
+
+ return wrapper
+
+
+def _in_memory_arrow_table_from_file(filename: str) -> pa.Table:
+ in_memory_stream = pa.input_stream(filename)
+ opened_stream = pa.ipc.open_stream(in_memory_stream)
+ pa_table = opened_stream.read_all()
+ return pa_table
+
+
+def _in_memory_arrow_table_from_buffer(buffer: pa.Buffer) -> pa.Table:
+ stream = pa.BufferReader(buffer)
+ opened_stream = pa.ipc.open_stream(stream)
+ table = opened_stream.read_all()
+ return table
+
+
+def _memory_mapped_record_batch_reader_from_file(filename: str) -> pa.RecordBatchStreamReader:
+ memory_mapped_stream = pa.memory_map(filename)
+ return pa.ipc.open_stream(memory_mapped_stream)
+
+
+def read_schema_from_file(filename: str) -> pa.Schema:
+ """
+ Infer arrow table schema from file without loading whole file into memory.
+ Useful especially while having very big files.
+ """
+ with pa.memory_map(filename) as memory_mapped_stream:
+ schema = pa.ipc.open_stream(memory_mapped_stream).schema
+ return schema
+
+
+def _batch_arrow_table(table: pa.Table) -> pa.Table:
+ batched_columns = []
+ for column_name in table.column_names:
+ column = table[column_name].combine_chunks()
+ offsets = pa.array([0, len(column)], type=pa.int32())
+ batched_columns.append(pa.ListArray.from_arrays(offsets, column))
+ return pa.Table.from_arrays(batched_columns, names=table.column_names)
+
+
+def _batch_accumulate_arrow_table_by_columns(
+ table: pa.Table, indices: list[int], columns: tuple[str], tables_accumulator: list[pa.Table], length: Optional[int]
+) -> pa.Table:
+ accumulate_last_batch = length is None or indices[-1] + 1 < length
+ # keep accumulating if key is the same, otherwise include the accumulated tables
+ if tables_accumulator and accumulate_last_batch:
+ for column in columns:
+ if any(pc.not_equal(table[column], tables_accumulator[0][column][0]).to_pylist()):
+ break
+ else:
+ tables_accumulator.append(table)
+ empty_batched_table = pa.Table.from_arrays(
+ [
+ pa.ListArray.from_arrays(pa.array([0], type=pa.int32()), table[column].chunk(0))
+ for column in table.column_names
+ ],
+ names=table.column_names,
+ )
+ return empty_batched_table
+ table = pa.concat_tables(tables_accumulator + [table])
+ tables_accumulator[:] = []
+ if len(table) == 0:
+ return table
+ # cut the table per key, i.e. when the columns change value
+ if len(table) > 1:
+ cut_array = pc.not_equal(table[columns[0]][1:], table[columns[0]][:-1])
+ for column in columns[1:]:
+ cut_array = pc.or_(cut_array, pc.not_equal(table[column][1:], table[column][:-1]))
+ else:
+ cut_array = pa.array([], type=pa.uint64())
+ offsets = pc.indices_nonzero(cut_array)
+ # make the batched table
+ offsets = pc.add(1, offsets)
+ offsets = pa.concat_arrays([pa.array([0], type=pa.int32()), offsets.cast(pa.int32())])
+ if not accumulate_last_batch:
+ offsets = pa.concat_arrays([offsets, pa.array([len(table)], type=pa.int32())])
+ batched_columns = []
+ for column_name in table.column_names:
+ column = table[column_name].combine_chunks()
+ batched_columns.append(pa.ListArray.from_arrays(offsets, column))
+ batched_table = pa.Table.from_arrays(batched_columns, names=table.column_names)
+ # add the last batch to the accumulator since it might not be full yet
+ if accumulate_last_batch:
+ last_offset = offsets[-1].as_py()
+ tables_accumulator.append(table.slice(last_offset, len(table) - last_offset))
+ return batched_table
+
+
+def _memory_mapped_arrow_table_from_file(filename: str) -> pa.Table:
+ opened_stream = _memory_mapped_record_batch_reader_from_file(filename)
+ pa_table = opened_stream.read_all()
+ return pa_table
+
+
+def _deepcopy(x, memo: dict):
+ """deepcopy a regular class instance"""
+ cls = x.__class__
+ result = cls.__new__(cls)
+ memo[id(x)] = result
+ for k, v in x.__dict__.items():
+ setattr(result, k, copy.deepcopy(v, memo))
+ return result
+
+
+def _interpolation_search(arr: list[int], x: int) -> int:
+ """
+ Return the position i of a sorted array so that arr[i] <= x < arr[i+1]
+
+ Args:
+ arr (`List[int]`): non-empty sorted list of integers
+ x (`int`): query
+
+ Returns:
+ `int`: the position i so that arr[i] <= x < arr[i+1]
+
+ Raises:
+ `IndexError`: if the array is empty or if the query is outside the array values
+ """
+ i, j = 0, len(arr) - 1
+ while i < j and arr[i] <= x < arr[j]:
+ k = i + ((j - i) * (x - arr[i]) // (arr[j] - arr[i]))
+ if arr[k] <= x < arr[k + 1]:
+ return k
+ elif arr[k] < x:
+ i, j = k + 1, j
+ else:
+ i, j = i, k
+ raise IndexError(f"Invalid query '{x}' for size {arr[-1] if len(arr) else 'none'}.")
+
+
+class IndexedTableMixin:
+ def __init__(self, table: pa.Table):
+ self._schema: pa.Schema = table.schema
+ self._batches: list[pa.RecordBatch] = [
+ recordbatch for recordbatch in table.to_batches() if len(recordbatch) > 0
+ ]
+ self._offsets: np.ndarray = np.cumsum([0] + [len(b) for b in self._batches], dtype=np.int64)
+
+ def fast_gather(self, indices: Union[list[int], np.ndarray]) -> pa.Table:
+ """
+ Create a pa.Table by gathering the records at the records at the specified indices. Should be faster
+ than pa.concat_tables(table.fast_slice(int(i) % table.num_rows, 1) for i in indices) since NumPy can compute
+ the binary searches in parallel, highly optimized C
+ """
+ if not len(indices):
+ raise ValueError("Indices must be non-empty")
+ batch_indices = np.searchsorted(self._offsets, indices, side="right") - 1
+ return pa.Table.from_batches(
+ [
+ self._batches[batch_idx].slice(i - self._offsets[batch_idx], 1)
+ for batch_idx, i in zip(batch_indices, indices)
+ ],
+ schema=self._schema,
+ )
+
+ def fast_slice(self, offset=0, length=None) -> pa.Table:
+ """
+ Slice the Table using interpolation search.
+ The behavior is the same as `pyarrow.Table.slice` but it's significantly faster.
+
+ Interpolation search is used to find the start and end indexes of the batches we want to keep.
+ The batches to keep are then concatenated to form the sliced Table.
+ """
+ if offset < 0:
+ raise IndexError("Offset must be non-negative")
+ elif offset >= self._offsets[-1] or (length is not None and length <= 0):
+ return pa.Table.from_batches([], schema=self._schema)
+ i = _interpolation_search(self._offsets, offset)
+ if length is None or length + offset >= self._offsets[-1]:
+ batches = self._batches[i:]
+ batches[0] = batches[0].slice(offset - self._offsets[i])
+ else:
+ j = _interpolation_search(self._offsets, offset + length - 1)
+ batches = self._batches[i : j + 1]
+ batches[-1] = batches[-1].slice(0, offset + length - self._offsets[j])
+ batches[0] = batches[0].slice(offset - self._offsets[i])
+ return pa.Table.from_batches(batches, schema=self._schema)
+
+
+class Table(IndexedTableMixin):
+ """
+ Wraps a pyarrow Table by using composition.
+ This is the base class for `InMemoryTable`, `MemoryMappedTable` and `ConcatenationTable`.
+
+ It implements all the basic attributes/methods of the pyarrow Table class except
+ the Table transforms: `slice, filter, flatten, combine_chunks, cast, add_column,
+ append_column, remove_column, set_column, rename_columns` and `drop`.
+
+ The implementation of these methods differs for the subclasses.
+ """
+
+ def __init__(self, table: pa.Table):
+ super().__init__(table)
+ self.table = table
+
+ def __deepcopy__(self, memo: dict):
+ # arrow tables are immutable, so there's no need to copy self.table
+ # moreover calling deepcopy on a pyarrow table seems to make pa.total_allocated_bytes() decrease for some reason
+ # by adding it to the memo, self.table won't be copied
+ memo[id(self.table)] = self.table
+ # same for the recordbatches used by the index
+ memo[id(self._batches)] = list(self._batches)
+ return _deepcopy(self, memo)
+
+ def validate(self, *args, **kwargs):
+ """
+ Perform validation checks. An exception is raised if validation fails.
+
+ By default only cheap validation checks are run. Pass `full=True`
+ for thorough validation checks (potentially `O(n)`).
+
+ Args:
+ full (`bool`, defaults to `False`):
+ If `True`, run expensive checks, otherwise cheap checks only.
+
+ Raises:
+ `pa.lib.ArrowInvalid`: if validation fails
+ """
+ return self.table.validate(*args, **kwargs)
+
+ def equals(self, *args, **kwargs):
+ """
+ Check if contents of two tables are equal.
+
+ Args:
+ other ([`~datasets.table.Table`]):
+ Table to compare against.
+ check_metadata `bool`, defaults to `False`):
+ Whether schema metadata equality should be checked as well.
+
+ Returns:
+ `bool`
+ """
+ args = tuple(arg.table if isinstance(arg, Table) else arg for arg in args)
+ kwargs = {k: v.table if isinstance(v, Table) else v for k, v in kwargs}
+ return self.table.equals(*args, **kwargs)
+
+ def to_batches(self, *args, **kwargs):
+ """
+ Convert Table to list of (contiguous) `RecordBatch` objects.
+
+ Args:
+ max_chunksize (`int`, defaults to `None`):
+ Maximum size for `RecordBatch` chunks. Individual chunks may be
+ smaller depending on the chunk layout of individual columns.
+
+ Returns:
+ `List[pyarrow.RecordBatch]`
+ """
+ return self.table.to_batches(*args, **kwargs)
+
+ def to_pydict(self, *args, **kwargs):
+ """
+ Convert the Table to a `dict` or `OrderedDict`.
+
+ Returns:
+ `dict`
+ """
+ return self.table.to_pydict(*args, **kwargs)
+
+ def to_pylist(self, *args, **kwargs):
+ """
+ Convert the Table to a list
+
+ Returns:
+ `list`
+ """
+ return self.table.to_pylist(*args, **kwargs)
+
+ def to_pandas(self, *args, **kwargs):
+ """
+ Convert to a pandas-compatible NumPy array or DataFrame, as appropriate.
+
+ Args:
+ memory_pool (`MemoryPool`, defaults to `None`):
+ Arrow MemoryPool to use for allocations. Uses the default memory
+ pool is not passed.
+ strings_to_categorical (`bool`, defaults to `False`):
+ Encode string (UTF8) and binary types to `pandas.Categorical`.
+ categories (`list`, defaults to `empty`):
+ List of fields that should be returned as `pandas.Categorical`. Only
+ applies to table-like data structures.
+ zero_copy_only (`bool`, defaults to `False`):
+ Raise an `ArrowException` if this function call would require copying
+ the underlying data.
+ integer_object_nulls (`bool`, defaults to `False`):
+ Cast integers with nulls to objects.
+ date_as_object (`bool`, defaults to `True`):
+ Cast dates to objects. If `False`, convert to `datetime64[ns]` dtype.
+ timestamp_as_object (`bool`, defaults to `False`):
+ Cast non-nanosecond timestamps (`np.datetime64`) to objects. This is
+ useful if you have timestamps that don't fit in the normal date
+ range of nanosecond timestamps (1678 CE-2262 CE).
+ If `False`, all timestamps are converted to `datetime64[ns]` dtype.
+ use_threads (`bool`, defaults to `True`):
+ Whether to parallelize the conversion using multiple threads.
+ deduplicate_objects (`bool`, defaults to `False`):
+ Do not create multiple copies Python objects when created, to save
+ on memory use. Conversion will be slower.
+ ignore_metadata (`bool`, defaults to `False`):
+ If `True`, do not use the 'pandas' metadata to reconstruct the
+ DataFrame index, if present.
+ safe (`bool`, defaults to `True`):
+ For certain data types, a cast is needed in order to store the
+ data in a pandas DataFrame or Series (e.g. timestamps are always
+ stored as nanoseconds in pandas). This option controls whether it
+ is a safe cast or not.
+ split_blocks (`bool`, defaults to `False`):
+ If `True`, generate one internal "block" for each column when
+ creating a pandas.DataFrame from a `RecordBatch` or `Table`. While this
+ can temporarily reduce memory note that various pandas operations
+ can trigger "consolidation" which may balloon memory use.
+ self_destruct (`bool`, defaults to `False`):
+ EXPERIMENTAL: If `True`, attempt to deallocate the originating Arrow
+ memory while converting the Arrow object to pandas. If you use the
+ object after calling `to_pandas` with this option it will crash your
+ program.
+ types_mapper (`function`, defaults to `None`):
+ A function mapping a pyarrow DataType to a pandas `ExtensionDtype`.
+ This can be used to override the default pandas type for conversion
+ of built-in pyarrow types or in absence of `pandas_metadata` in the
+ Table schema. The function receives a pyarrow DataType and is
+ expected to return a pandas `ExtensionDtype` or `None` if the
+ default conversion should be used for that type. If you have
+ a dictionary mapping, you can pass `dict.get` as function.
+
+ Returns:
+ `pandas.Series` or `pandas.DataFrame`: `pandas.Series` or `pandas.DataFrame` depending on type of object
+ """
+ return self.table.to_pandas(*args, **kwargs)
+
+ def to_string(self, *args, **kwargs):
+ return self.table.to_string(*args, **kwargs)
+
+ def to_reader(self, max_chunksize: Optional[int] = None):
+ """
+ Convert the Table to a RecordBatchReader.
+
+ Note that this method is zero-copy, it merely exposes the same data under a different API.
+
+ Args:
+ max_chunksize (`int`, defaults to `None`)
+ Maximum size for RecordBatch chunks. Individual chunks may be smaller depending
+ on the chunk layout of individual columns.
+
+ Returns:
+ `pyarrow.RecordBatchReader`
+ """
+ return self.table.to_reader(max_chunksize=max_chunksize)
+
+ def field(self, *args, **kwargs):
+ """
+ Select a schema field by its column name or numeric index.
+
+ Args:
+ i (`Union[int, str]`):
+ The index or name of the field to retrieve.
+
+ Returns:
+ `pyarrow.Field`
+ """
+ return self.table.field(*args, **kwargs)
+
+ def column(self, *args, **kwargs):
+ """
+ Select a column by its column name, or numeric index.
+
+ Args:
+ i (`Union[int, str]`):
+ The index or name of the column to retrieve.
+
+ Returns:
+ `pyarrow.ChunkedArray`
+ """
+ return self.table.column(*args, **kwargs)
+
+ def itercolumns(self, *args, **kwargs):
+ """
+ Iterator over all columns in their numerical order.
+
+ Yields:
+ `pyarrow.ChunkedArray`
+ """
+ return self.table.itercolumns(*args, **kwargs)
+
+ @property
+ def schema(self):
+ """
+ Schema of the table and its columns.
+
+ Returns:
+ `pyarrow.Schema`
+ """
+ return self.table.schema
+
+ @property
+ def columns(self):
+ """
+ List of all columns in numerical order.
+
+ Returns:
+ `List[pa.ChunkedArray]`
+ """
+ return self.table.columns
+
+ @property
+ def num_columns(self):
+ """
+ Number of columns in this table.
+
+ Returns:
+ int
+ """
+ return self.table.num_columns
+
+ @property
+ def num_rows(self):
+ """
+ Number of rows in this table.
+
+ Due to the definition of a table, all columns have the same number of
+ rows.
+
+ Returns:
+ int
+ """
+ return self.table.num_rows
+
+ @property
+ def shape(self):
+ """
+ Dimensions of the table: (#rows, #columns).
+
+ Returns:
+ `(int, int)`: Number of rows and number of columns.
+ """
+ return self.table.shape
+
+ @property
+ def nbytes(self):
+ """
+ Total number of bytes consumed by the elements of the table.
+ """
+ return self.table.nbytes
+
+ @property
+ def column_names(self):
+ """
+ Names of the table's columns.
+ """
+ return self.table.column_names
+
+ def __eq__(self, other):
+ return self.equals(other)
+
+ def __getitem__(self, i):
+ return self.table[i]
+
+ def __len__(self):
+ return len(self.table)
+
+ def __repr__(self):
+ return self.table.__repr__().replace("pyarrow.Table", self.__class__.__name__)
+
+ def __str__(self):
+ return self.table.__str__().replace("pyarrow.Table", self.__class__.__name__)
+
+ def slice(self, *args, **kwargs):
+ """
+ Compute zero-copy slice of this Table.
+
+ Args:
+ offset (`int`, defaults to `0`):
+ Offset from start of table to slice.
+ length (`int`, defaults to `None`):
+ Length of slice (default is until end of table starting from
+ offset).
+
+ Returns:
+ `datasets.table.Table`
+ """
+ raise NotImplementedError()
+
+ def filter(self, *args, **kwargs):
+ """
+ Select records from a Table. See `pyarrow.compute.filter` for full usage.
+ """
+ raise NotImplementedError()
+
+ def flatten(self, *args, **kwargs):
+ """
+ Flatten this Table. Each column with a struct type is flattened
+ into one column per struct field. Other columns are left unchanged.
+
+ Args:
+ memory_pool (`MemoryPool`, defaults to `None`):
+ For memory allocations, if required, otherwise use default pool.
+
+ Returns:
+ `datasets.table.Table`
+ """
+ raise NotImplementedError()
+
+ def combine_chunks(self, *args, **kwargs):
+ """
+ Make a new table by combining the chunks this table has.
+
+ All the underlying chunks in the `ChunkedArray` of each column are
+ concatenated into zero or one chunk.
+
+ Args:
+ memory_pool (`MemoryPool`, defaults to `None`):
+ For memory allocations, if required, otherwise use default pool.
+
+ Returns:
+ `datasets.table.Table`
+ """
+ raise NotImplementedError()
+
+ def cast(self, *args, **kwargs):
+ """
+ Cast table values to another schema.
+
+ Args:
+ target_schema (`Schema`):
+ Schema to cast to, the names and order of fields must match.
+ safe (`bool`, defaults to `True`):
+ Check for overflows or other unsafe conversions.
+
+ Returns:
+ `datasets.table.Table`
+ """
+ raise NotImplementedError()
+
+ def replace_schema_metadata(self, *args, **kwargs):
+ """
+ EXPERIMENTAL: Create shallow copy of table by replacing schema
+ key-value metadata with the indicated new metadata (which may be None,
+ which deletes any existing metadata
+
+ Args:
+ metadata (`dict`, defaults to `None`):
+
+ Returns:
+ `datasets.table.Table`: shallow_copy
+ """
+ raise NotImplementedError()
+
+ def add_column(self, *args, **kwargs):
+ """
+ Add column to Table at position.
+
+ A new table is returned with the column added, the original table
+ object is left unchanged.
+
+ Args:
+ i (`int`):
+ Index to place the column at.
+ field_ (`Union[str, pyarrow.Field]`):
+ If a string is passed then the type is deduced from the column
+ data.
+ column (`Union[pyarrow.Array, List[pyarrow.Array]]`):
+ Column data.
+
+ Returns:
+ `datasets.table.Table`: New table with the passed column added.
+ """
+ raise NotImplementedError()
+
+ def append_column(self, *args, **kwargs):
+ """
+ Append column at end of columns.
+
+ Args:
+ field_ (`Union[str, pyarrow.Field]`):
+ If a string is passed then the type is deduced from the column
+ data.
+ column (`Union[pyarrow.Array, List[pyarrow.Array]]`):
+ Column data.
+
+ Returns:
+ `datasets.table.Table`: New table with the passed column added.
+ """
+ raise NotImplementedError()
+
+ def remove_column(self, *args, **kwargs):
+ """
+ Create new Table with the indicated column removed.
+
+ Args:
+ i (`int`):
+ Index of column to remove.
+
+ Returns:
+ `datasets.table.Table`: New table without the column.
+ """
+ raise NotImplementedError()
+
+ def set_column(self, *args, **kwargs):
+ """
+ Replace column in Table at position.
+
+ Args:
+ i (`int`):
+ Index to place the column at.
+ field_ (`Union[str, pyarrow.Field]`):
+ If a string is passed then the type is deduced from the column
+ data.
+ column (`Union[pyarrow.Array, List[pyarrow.Array]]`):
+ Column data.
+
+ Returns:
+ `datasets.table.Table`: New table with the passed column set.
+ """
+ raise NotImplementedError()
+
+ def rename_columns(self, *args, **kwargs):
+ """
+ Create new table with columns renamed to provided names.
+ """
+ raise NotImplementedError()
+
+ def drop(self, *args, **kwargs):
+ """
+ Drop one or more columns and return a new table.
+
+ Args:
+ columns (`List[str]`):
+ List of field names referencing existing columns.
+
+ Raises:
+ `KeyError` : if any of the passed columns name are not existing.
+
+ Returns:
+ `datasets.table.Table`: New table without the columns.
+ """
+ raise NotImplementedError()
+
+ def select(self, *args, **kwargs):
+ """
+ Select columns of the table.
+
+ Returns a new table with the specified columns, and metadata preserved.
+
+ Args:
+ columns (:obj:`Union[List[str], List[int]]`):
+ The column names or integer indices to select.
+
+ Returns:
+ `datasets.table.Table`: table with only a subset of the columns
+ """
+ raise NotImplementedError()
+
+
+class TableBlock(Table):
+ """
+ `TableBlock` is the allowed class inside a `ConcanetationTable`.
+ Only `MemoryMappedTable` and `InMemoryTable` are `TableBlock`.
+ This is because we don't want a `ConcanetationTable` made out of other `ConcanetationTables`.
+ """
+
+ pass
+
+
+class InMemoryTable(TableBlock):
+ """
+ The table is said in-memory when it is loaded into the user's RAM.
+
+ Pickling it does copy all the data using memory.
+ Its implementation is simple and uses the underlying pyarrow Table methods directly.
+
+ This is different from the `MemoryMapped` table, for which pickling doesn't copy all the
+ data in memory. For a `MemoryMapped`, unpickling instead reloads the table from the disk.
+
+ `InMemoryTable` must be used when data fit in memory, while `MemoryMapped` are reserved for
+ data bigger than memory or when you want the memory footprint of your application to
+ stay low.
+ """
+
+ @classmethod
+ def from_file(cls, filename: str):
+ table = _in_memory_arrow_table_from_file(filename)
+ return cls(table)
+
+ @classmethod
+ def from_buffer(cls, buffer: pa.Buffer):
+ table = _in_memory_arrow_table_from_buffer(buffer)
+ return cls(table)
+
+ @classmethod
+ def from_pandas(cls, *args, **kwargs):
+ """
+ Convert pandas.DataFrame to an Arrow Table.
+
+ The column types in the resulting Arrow Table are inferred from the
+ dtypes of the pandas.Series in the DataFrame. In the case of non-object
+ Series, the NumPy dtype is translated to its Arrow equivalent. In the
+ case of `object`, we need to guess the datatype by looking at the
+ Python objects in this Series.
+
+ Be aware that Series of the `object` dtype don't carry enough
+ information to always lead to a meaningful Arrow type. In the case that
+ we cannot infer a type, e.g. because the DataFrame is of length 0 or
+ the Series only contains `None/nan` objects, the type is set to
+ null. This behavior can be avoided by constructing an explicit schema
+ and passing it to this function.
+
+ Args:
+ df (`pandas.DataFrame`):
+ schema (`pyarrow.Schema`, *optional*):
+ The expected schema of the Arrow Table. This can be used to
+ indicate the type of columns if we cannot infer it automatically.
+ If passed, the output will have exactly this schema. Columns
+ specified in the schema that are not found in the DataFrame columns
+ or its index will raise an error. Additional columns or index
+ levels in the DataFrame which are not specified in the schema will
+ be ignored.
+ preserve_index (`bool`, *optional*):
+ Whether to store the index as an additional column in the resulting
+ `Table`. The default of None will store the index as a column,
+ except for RangeIndex which is stored as metadata only. Use
+ `preserve_index=True` to force it to be stored as a column.
+ nthreads (`int`, defaults to `None` (may use up to system CPU count threads))
+ If greater than 1, convert columns to Arrow in parallel using
+ indicated number of threads.
+ columns (`List[str]`, *optional*):
+ List of column to be converted. If `None`, use all columns.
+ safe (`bool`, defaults to `True`):
+ Check for overflows or other unsafe conversions,
+
+ Returns:
+ `datasets.table.Table`:
+
+ Examples:
+ ```python
+ >>> import pandas as pd
+ >>> import pyarrow as pa
+ >>> df = pd.DataFrame({
+ ... 'int': [1, 2],
+ ... 'str': ['a', 'b']
+ ... })
+ >>> pa.Table.from_pandas(df)
+
+ ```
+ """
+ return cls(pa.Table.from_pandas(*args, **kwargs))
+
+ @classmethod
+ def from_arrays(cls, *args, **kwargs):
+ """
+ Construct a Table from Arrow arrays.
+
+ Args:
+ arrays (`List[Union[pyarrow.Array, pyarrow.ChunkedArray]]`):
+ Equal-length arrays that should form the table.
+ names (`List[str]`, *optional*):
+ Names for the table columns. If not passed, schema must be passed.
+ schema (`Schema`, defaults to `None`):
+ Schema for the created table. If not passed, names must be passed.
+ metadata (`Union[dict, Mapping]`, defaults to `None`):
+ Optional metadata for the schema (if inferred).
+
+ Returns:
+ `datasets.table.Table`
+ """
+ return cls(pa.Table.from_arrays(*args, **kwargs))
+
+ @classmethod
+ def from_pydict(cls, *args, **kwargs):
+ """
+ Construct a Table from Arrow arrays or columns.
+
+ Args:
+ mapping (`Union[dict, Mapping]`):
+ A mapping of strings to Arrays or Python lists.
+ schema (`Schema`, defaults to `None`):
+ If not passed, will be inferred from the Mapping values
+ metadata (`Union[dict, Mapping]`, defaults to `None`):
+ Optional metadata for the schema (if inferred).
+
+ Returns:
+ `datasets.table.Table`
+ """
+ return cls(pa.Table.from_pydict(*args, **kwargs))
+
+ @classmethod
+ def from_pylist(cls, mapping, *args, **kwargs):
+ """
+ Construct a Table from list of rows / dictionaries.
+
+ Args:
+ mapping (`List[dict]`):
+ A mapping of strings to row values.
+ schema (`Schema`, defaults to `None`):
+ If not passed, will be inferred from the Mapping values
+ metadata (`Union[dict, Mapping]`, defaults to `None`):
+ Optional metadata for the schema (if inferred).
+
+ Returns:
+ `datasets.table.Table`
+ """
+ return cls(pa.Table.from_pylist(mapping, *args, **kwargs))
+
+ @classmethod
+ def from_batches(cls, *args, **kwargs):
+ """
+ Construct a Table from a sequence or iterator of Arrow `RecordBatches`.
+
+ Args:
+ batches (`Union[Sequence[pyarrow.RecordBatch], Iterator[pyarrow.RecordBatch]]`):
+ Sequence of `RecordBatch` to be converted, all schemas must be equal.
+ schema (`Schema`, defaults to `None`):
+ If not passed, will be inferred from the first `RecordBatch`.
+
+ Returns:
+ `datasets.table.Table`:
+ """
+ return cls(pa.Table.from_batches(*args, **kwargs))
+
+ def slice(self, offset=0, length=None):
+ """
+ Compute zero-copy slice of this Table.
+
+ Args:
+ offset (`int`, defaults to `0`):
+ Offset from start of table to slice.
+ length (`int`, defaults to `None`):
+ Length of slice (default is until end of table starting from
+ offset).
+
+ Returns:
+ `datasets.table.Table`
+ """
+ # Use fast slicing here
+ return InMemoryTable(self.fast_slice(offset=offset, length=length))
+
+ def filter(self, *args, **kwargs):
+ """
+ Select records from a Table. See `pyarrow.compute.filter` for full usage.
+ """
+ return InMemoryTable(self.table.filter(*args, **kwargs))
+
+ def flatten(self, *args, **kwargs):
+ """
+ Flatten this Table. Each column with a struct type is flattened
+ into one column per struct field. Other columns are left unchanged.
+
+ Args:
+ memory_pool (`MemoryPool`, defaults to `None`):
+ For memory allocations, if required, otherwise use default pool.
+
+ Returns:
+ `datasets.table.Table`
+ """
+ return InMemoryTable(table_flatten(self.table, *args, **kwargs))
+
+ def combine_chunks(self, *args, **kwargs):
+ """
+ Make a new table by combining the chunks this table has.
+
+ All the underlying chunks in the `ChunkedArray` of each column are
+ concatenated into zero or one chunk.
+
+ Args:
+ memory_pool (`MemoryPool`, defaults to `None`):
+ For memory allocations, if required, otherwise use default pool.
+
+ Returns:
+ `datasets.table.Table`
+ """
+ return InMemoryTable(self.table.combine_chunks(*args, **kwargs))
+
+ def cast(self, *args, **kwargs):
+ """
+ Cast table values to another schema.
+
+ Args:
+ target_schema (`Schema`):
+ Schema to cast to, the names and order of fields must match.
+ safe (`bool`, defaults to `True`):
+ Check for overflows or other unsafe conversions.
+
+ Returns:
+ `datasets.table.Table`
+ """
+ return InMemoryTable(table_cast(self.table, *args, **kwargs))
+
+ def replace_schema_metadata(self, *args, **kwargs):
+ """
+ EXPERIMENTAL: Create shallow copy of table by replacing schema
+ key-value metadata with the indicated new metadata (which may be `None`,
+ which deletes any existing metadata).
+
+ Args:
+ metadata (`dict`, defaults to `None`):
+
+ Returns:
+ `datasets.table.Table`: shallow_copy
+ """
+ return InMemoryTable(self.table.replace_schema_metadata(*args, **kwargs))
+
+ def add_column(self, *args, **kwargs):
+ """
+ Add column to Table at position.
+
+ A new table is returned with the column added, the original table
+ object is left unchanged.
+
+ Args:
+ i (`int`):
+ Index to place the column at.
+ field_ (`Union[str, pyarrow.Field]`):
+ If a string is passed then the type is deduced from the column
+ data.
+ column (`Union[pyarrow.Array, List[pyarrow.Array]]`):
+ Column data.
+
+ Returns:
+ `datasets.table.Table`: New table with the passed column added.
+ """
+ return InMemoryTable(self.table.add_column(*args, **kwargs))
+
+ def append_column(self, *args, **kwargs):
+ """
+ Append column at end of columns.
+
+ Args:
+ field_ (`Union[str, pyarrow.Field]`):
+ If a string is passed then the type is deduced from the column
+ data.
+ column (`Union[pyarrow.Array, List[pyarrow.Array]]`):
+ Column data.
+
+ Returns:
+ `datasets.table.Table`:
+ New table with the passed column added.
+ """
+ return InMemoryTable(self.table.append_column(*args, **kwargs))
+
+ def remove_column(self, *args, **kwargs):
+ """
+ Create new Table with the indicated column removed.
+
+ Args:
+ i (`int`):
+ Index of column to remove.
+
+ Returns:
+ `datasets.table.Table`:
+ New table without the column.
+ """
+ return InMemoryTable(self.table.remove_column(*args, **kwargs))
+
+ def set_column(self, *args, **kwargs):
+ """
+ Replace column in Table at position.
+
+ Args:
+ i (`int`):
+ Index to place the column at.
+ field_ (`Union[str, pyarrow.Field]`):
+ If a string is passed then the type is deduced from the column
+ data.
+ column (`Union[pyarrow.Array, List[pyarrow.Array]]`):
+ Column data.
+
+ Returns:
+ `datasets.table.Table`:
+ New table with the passed column set.
+ """
+ return InMemoryTable(self.table.set_column(*args, **kwargs))
+
+ def rename_columns(self, *args, **kwargs):
+ """
+ Create new table with columns renamed to provided names.
+ """
+ return InMemoryTable(self.table.rename_columns(*args, **kwargs))
+
+ def drop(self, *args, **kwargs):
+ """
+ Drop one or more columns and return a new table.
+
+ Args:
+ columns (`List[str]`):
+ List of field names referencing existing columns.
+
+ Raises:
+ `KeyError` : if any of the passed columns name are not existing.
+
+ Returns:
+ `datasets.table.Table`:
+ New table without the columns.
+ """
+ return InMemoryTable(self.table.drop(*args, **kwargs))
+
+ def select(self, *args, **kwargs):
+ """
+ Select columns of the table.
+
+ Returns a new table with the specified columns, and metadata preserved.
+
+ Args:
+ columns (:obj:`Union[List[str], List[int]]`):
+ The column names or integer indices to select.
+
+ Returns:
+ :class:`datasets.table.Table`: New table with the specified columns, and metadata preserved.
+ """
+ return InMemoryTable(self.table.select(*args, **kwargs))
+
+
+# The MemoryMappedTable needs replays to properly reload tables from the disk
+Replay = tuple[str, tuple, dict]
+
+
+class MemoryMappedTable(TableBlock):
+ """
+ The table is said memory mapped when it doesn't use the user's RAM but loads the data
+ from the disk instead.
+
+ Pickling it doesn't copy the data into memory.
+ Instead, only the path to the memory mapped arrow file is pickled, as well as the list
+ of transforms to "replay" when reloading the table from the disk.
+
+ Its implementation requires to store an history of all the transforms that were applied
+ to the underlying pyarrow Table, so that they can be "replayed" when reloading the Table
+ from the disk.
+
+ This is different from the `InMemoryTable` table, for which pickling does copy all the
+ data in memory.
+
+ `InMemoryTable` must be used when data fit in memory, while `MemoryMapped` are reserved for
+ data bigger than memory or when you want the memory footprint of your application to
+ stay low.
+ """
+
+ def __init__(self, table: pa.Table, path: str, replays: Optional[list[Replay]] = None):
+ super().__init__(table)
+ self.path = os.path.abspath(path)
+ self.replays: list[Replay] = replays if replays is not None else []
+
+ @classmethod
+ def from_file(cls, filename: str, replays=None):
+ table = _memory_mapped_arrow_table_from_file(filename)
+ table = cls._apply_replays(table, replays)
+ return cls(table, filename, replays)
+
+ def __getstate__(self):
+ return {"path": self.path, "replays": self.replays}
+
+ def __setstate__(self, state):
+ path = state["path"]
+ replays = state["replays"]
+ table = _memory_mapped_arrow_table_from_file(path)
+ table = self._apply_replays(table, replays)
+ MemoryMappedTable.__init__(self, table, path=path, replays=replays)
+
+ @staticmethod
+ def _apply_replays(table: pa.Table, replays: Optional[list[Replay]] = None) -> pa.Table:
+ if replays is not None:
+ for name, args, kwargs in replays:
+ if name == "cast":
+ table = table_cast(table, *args, **kwargs)
+ elif name == "flatten":
+ table = table_flatten(table, *args, **kwargs)
+ else:
+ table = getattr(table, name)(*args, **kwargs)
+ return table
+
+ def _append_replay(self, replay: Replay) -> list[Replay]:
+ replays = copy.deepcopy(self.replays)
+ replays.append(replay)
+ return replays
+
+ def slice(self, offset=0, length=None):
+ """
+ Compute zero-copy slice of this Table.
+
+ Args:
+ offset (`int`, defaults to `0`):
+ Offset from start of table to slice.
+ length (`int`, defaults to `None`):
+ Length of slice (default is until end of table starting from
+ offset).
+
+ Returns:
+ `datasets.table.Table`
+ """
+ replay = ("slice", (offset, length), {})
+ replays = self._append_replay(replay)
+ # Use fast slicing here
+ return MemoryMappedTable(self.fast_slice(offset=offset, length=length), self.path, replays)
+
+ def filter(self, *args, **kwargs):
+ """
+ Select records from a Table. See `pyarrow.compute.filter` for full usage.
+ """
+ replay = ("filter", copy.deepcopy(args), copy.deepcopy(kwargs))
+ replays = self._append_replay(replay)
+ return MemoryMappedTable(self.table.filter(*args, **kwargs), self.path, replays)
+
+ def flatten(self, *args, **kwargs):
+ """
+ Flatten this Table. Each column with a struct type is flattened
+ into one column per struct field. Other columns are left unchanged.
+
+ Args:
+ memory_pool (`MemoryPool`, defaults to `None`):
+ For memory allocations, if required, otherwise use default pool.
+
+ Returns:
+ `datasets.table.Table`
+ """
+ replay = ("flatten", copy.deepcopy(args), copy.deepcopy(kwargs))
+ replays = self._append_replay(replay)
+ return MemoryMappedTable(table_flatten(self.table, *args, **kwargs), self.path, replays)
+
+ def combine_chunks(self, *args, **kwargs):
+ """
+ Make a new table by combining the chunks this table has.
+
+ All the underlying chunks in the ChunkedArray of each column are
+ concatenated into zero or one chunk.
+
+ Args:
+ memory_pool (`MemoryPool`, defaults to `None`):
+ For memory allocations, if required, otherwise use default pool.
+
+ Returns:
+ `datasets.table.Table`
+ """
+ replay = ("combine_chunks", copy.deepcopy(args), copy.deepcopy(kwargs))
+ replays = self._append_replay(replay)
+ return MemoryMappedTable(self.table.combine_chunks(*args, **kwargs), self.path, replays)
+
+ def cast(self, *args, **kwargs):
+ """
+ Cast table values to another schema
+
+ Args:
+ target_schema (`Schema`):
+ Schema to cast to, the names and order of fields must match.
+ safe (`bool`, defaults to `True`):
+ Check for overflows or other unsafe conversions.
+
+ Returns:
+ `datasets.table.Table`
+ """
+ replay = ("cast", copy.deepcopy(args), copy.deepcopy(kwargs))
+ replays = self._append_replay(replay)
+ return MemoryMappedTable(table_cast(self.table, *args, **kwargs), self.path, replays)
+
+ def replace_schema_metadata(self, *args, **kwargs):
+ """
+ EXPERIMENTAL: Create shallow copy of table by replacing schema
+ key-value metadata with the indicated new metadata (which may be None,
+ which deletes any existing metadata.
+
+ Args:
+ metadata (`dict`, defaults to `None`):
+
+ Returns:
+ `datasets.table.Table`: shallow_copy
+ """
+ replay = ("replace_schema_metadata", copy.deepcopy(args), copy.deepcopy(kwargs))
+ replays = self._append_replay(replay)
+ return MemoryMappedTable(self.table.replace_schema_metadata(*args, **kwargs), self.path, replays)
+
+ def add_column(self, *args, **kwargs):
+ """
+ Add column to Table at position.
+
+ A new table is returned with the column added, the original table
+ object is left unchanged.
+
+ Args:
+ i (`int`):
+ Index to place the column at.
+ field_ (`Union[str, pyarrow.Field]`):
+ If a string is passed then the type is deduced from the column
+ data.
+ column (`Union[pyarrow.Array, List[pyarrow.Array]]`):
+ Column data.
+
+ Returns:
+ `datasets.table.Table`: New table with the passed column added.
+ """
+ replay = ("add_column", copy.deepcopy(args), copy.deepcopy(kwargs))
+ replays = self._append_replay(replay)
+ return MemoryMappedTable(self.table.add_column(*args, **kwargs), self.path, replays)
+
+ def append_column(self, *args, **kwargs):
+ """
+ Append column at end of columns.
+
+ Args:
+ field_ (`Union[str, pyarrow.Field]`):
+ If a string is passed then the type is deduced from the column
+ data.
+ column (`Union[pyarrow.Array, List[pyarrow.Array]]`):
+ Column data.
+
+ Returns:
+ `datasets.table.Table`:
+ New table with the passed column added.
+ """
+ replay = ("append_column", copy.deepcopy(args), copy.deepcopy(kwargs))
+ replays = self._append_replay(replay)
+ return MemoryMappedTable(self.table.append_column(*args, **kwargs), self.path, replays)
+
+ def remove_column(self, *args, **kwargs):
+ """
+ Create new Table with the indicated column removed.
+
+ Args:
+ i (`int`):
+ Index of column to remove.
+
+ Returns:
+ `datasets.table.Table`:
+ New table without the column.
+ """
+ replay = ("remove_column", copy.deepcopy(args), copy.deepcopy(kwargs))
+ replays = self._append_replay(replay)
+ return MemoryMappedTable(self.table.remove_column(*args, **kwargs), self.path, replays)
+
+ def set_column(self, *args, **kwargs):
+ """
+ Replace column in Table at position.
+
+ Args:
+ i (`int`):
+ Index to place the column at.
+ field_ (`Union[str, pyarrow.Field]`):
+ If a string is passed then the type is deduced from the column
+ data.
+ column (`Union[pyarrow.Array, List[pyarrow.Array]]`):
+ Column data.
+
+ Returns:
+ `datasets.table.Table`:
+ New table with the passed column set.
+ """
+ replay = ("set_column", copy.deepcopy(args), copy.deepcopy(kwargs))
+ replays = self._append_replay(replay)
+ return MemoryMappedTable(self.table.set_column(*args, **kwargs), self.path, replays)
+
+ def rename_columns(self, *args, **kwargs):
+ """
+ Create new table with columns renamed to provided names.
+ """
+ replay = ("rename_columns", copy.deepcopy(args), copy.deepcopy(kwargs))
+ replays = self._append_replay(replay)
+ return MemoryMappedTable(self.table.rename_columns(*args, **kwargs), self.path, replays)
+
+ def drop(self, *args, **kwargs):
+ """
+ Drop one or more columns and return a new table.
+
+ Args:
+ columns (`List[str]`):
+ List of field names referencing existing columns.
+
+ Raises:
+ `KeyError` : if any of the passed columns name are not existing.
+
+ Returns:
+ `datasets.table.Table`:
+ New table without the columns.
+ """
+ replay = ("drop", copy.deepcopy(args), copy.deepcopy(kwargs))
+ replays = self._append_replay(replay)
+ return MemoryMappedTable(self.table.drop(*args, **kwargs), self.path, replays)
+
+ def select(self, *args, **kwargs):
+ """
+ Select columns of the table.
+
+ Returns a new table with the specified columns, and metadata preserved.
+
+ Args:
+ columns (:obj:`Union[List[str], List[int]]`):
+ The column names or integer indices to select.
+
+ Returns:
+ :class:`datasets.table.Table`: New table with the specified columns, and metadata preserved.
+ """
+ replay = ("select", copy.deepcopy(args), copy.deepcopy(kwargs))
+ replays = self._append_replay(replay)
+ return MemoryMappedTable(self.table.select(*args, **kwargs), self.path, replays)
+
+
+# A ConcatenationTable is the concatenation of several tables.
+# The ``blocks`` attributes stores a list of list of blocks.
+# The first axis concatenates the tables along the axis 0 (it appends rows),
+# while the second axis concatenates tables along the axis 1 (it appends columns).
+TableBlockContainer = TypeVar("TableBlockContainer", TableBlock, list[TableBlock], list[list[TableBlock]])
+
+
+class ConcatenationTable(Table):
+ """
+ The table comes from the concatenation of several tables called blocks.
+ It enables concatenation on both axis 0 (append rows) and axis 1 (append columns).
+
+ The underlying tables are called "blocks" and can be either `InMemoryTable`
+ or `MemoryMappedTable` objects.
+ This allows to combine tables that come from memory or that are memory mapped.
+ When a `ConcatenationTable` is pickled, then each block is pickled:
+ - the `InMemoryTable` objects are pickled by copying all the data in memory.
+ - the MemoryMappedTable objects are pickled without copying the data into memory.
+ Instead, only the path to the memory mapped arrow file is pickled, as well as the list
+ of transforms to "replays" when reloading the table from the disk.
+
+ Its implementation requires to store each block separately.
+ The `blocks` attributes stores a list of list of blocks.
+ The first axis concatenates the tables along the axis 0 (it appends rows),
+ while the second axis concatenates tables along the axis 1 (it appends columns).
+
+ If some columns are missing when concatenating on axis 0, they are filled with null values.
+ This is done using `pyarrow.concat_tables(tables, promote=True)`.
+
+ You can access the fully combined table by accessing the `ConcatenationTable.table` attribute,
+ and the blocks by accessing the `ConcatenationTable.blocks` attribute.
+ """
+
+ def __init__(self, table: pa.Table, blocks: list[list[TableBlock]]):
+ super().__init__(table)
+ self.blocks = blocks
+ # Check that all the blocks have the right type.
+ # Only InMemoryTable and MemoryMappedTable are allowed.
+ for subtables in blocks:
+ for subtable in subtables:
+ if not isinstance(subtable, TableBlock):
+ raise TypeError(
+ "The blocks of a ConcatenationTable must be InMemoryTable or MemoryMappedTable objects"
+ f", but got {_short_str(subtable)}."
+ )
+
+ def __getstate__(self):
+ return {"blocks": self.blocks, "schema": self.table.schema}
+
+ def __setstate__(self, state):
+ blocks = state["blocks"]
+ schema = state["schema"]
+ table = self._concat_blocks_horizontally_and_vertically(blocks)
+ if schema is not None and table.schema != schema:
+ # We fix the columns by concatenating with an empty table with the right columns
+ empty_table = pa.Table.from_batches([], schema=schema)
+ # We set promote_options="default" to fill missing columns with null values
+ table = pa.concat_tables([table, empty_table], promote_options="default")
+ ConcatenationTable.__init__(self, table, blocks=blocks)
+
+ @staticmethod
+ def _concat_blocks(blocks: list[Union[TableBlock, pa.Table]], axis: int = 0) -> pa.Table:
+ pa_tables = [table.table if hasattr(table, "table") else table for table in blocks]
+ if axis == 0:
+ # We set promote_options="default" to fill missing columns with null values
+ return pa.concat_tables(pa_tables, promote_options="default")
+ elif axis == 1:
+ for i, table in enumerate(pa_tables):
+ if i == 0:
+ pa_table = table
+ else:
+ for name, col in zip(table.column_names, table.columns):
+ pa_table = pa_table.append_column(name, col)
+ return pa_table
+ else:
+ raise ValueError("'axis' must be either 0 or 1")
+
+ @classmethod
+ def _concat_blocks_horizontally_and_vertically(cls, blocks: list[list[TableBlock]]) -> pa.Table:
+ pa_tables_to_concat_vertically = []
+ for i, tables in enumerate(blocks):
+ if not tables:
+ continue
+ pa_table_horizontally_concatenated = cls._concat_blocks(tables, axis=1)
+ pa_tables_to_concat_vertically.append(pa_table_horizontally_concatenated)
+ return cls._concat_blocks(pa_tables_to_concat_vertically, axis=0)
+
+ @classmethod
+ def _merge_blocks(cls, blocks: TableBlockContainer, axis: Optional[int] = None) -> TableBlockContainer:
+ if axis is not None:
+ merged_blocks = []
+ for is_in_memory, block_group in groupby(blocks, key=lambda x: isinstance(x, InMemoryTable)):
+ if is_in_memory:
+ block_group = [InMemoryTable(cls._concat_blocks(list(block_group), axis=axis))]
+ merged_blocks += list(block_group)
+ else: # both
+ merged_blocks = [cls._merge_blocks(row_block, axis=1) for row_block in blocks]
+ if all(len(row_block) == 1 for row_block in merged_blocks):
+ merged_blocks = cls._merge_blocks(
+ [block for row_block in merged_blocks for block in row_block], axis=0
+ )
+ return merged_blocks
+
+ @classmethod
+ def _consolidate_blocks(cls, blocks: TableBlockContainer) -> TableBlockContainer:
+ if isinstance(blocks, TableBlock):
+ return blocks
+ elif isinstance(blocks[0], TableBlock):
+ return cls._merge_blocks(blocks, axis=0)
+ else:
+ return cls._merge_blocks(blocks)
+
+ @classmethod
+ def from_blocks(cls, blocks: TableBlockContainer) -> "ConcatenationTable":
+ blocks = cls._consolidate_blocks(blocks)
+ if isinstance(blocks, TableBlock):
+ table = blocks
+ return cls(table.table, [[table]])
+ elif isinstance(blocks[0], TableBlock):
+ table = cls._concat_blocks(blocks, axis=0)
+ blocks = [[t] for t in blocks]
+ return cls(table, blocks)
+ else:
+ table = cls._concat_blocks_horizontally_and_vertically(blocks)
+ return cls(table, blocks)
+
+ @classmethod
+ def from_tables(cls, tables: list[Union[pa.Table, Table]], axis: int = 0) -> "ConcatenationTable":
+ """Create `ConcatenationTable` from list of tables.
+
+ Args:
+ tables (list of `Table` or list of `pyarrow.Table`):
+ List of tables.
+ axis (`{0, 1}`, defaults to `0`, meaning over rows):
+ Axis to concatenate over, where `0` means over rows (vertically) and `1` means over columns
+ (horizontally).
+
+
+ """
+
+ def to_blocks(table: Union[pa.Table, Table]) -> list[list[TableBlock]]:
+ if isinstance(table, pa.Table):
+ return [[InMemoryTable(table)]]
+ elif isinstance(table, ConcatenationTable):
+ return copy.deepcopy(table.blocks)
+ else:
+ return [[table]]
+
+ def _slice_row_block(row_block: list[TableBlock], length: int) -> tuple[list[TableBlock], list[TableBlock]]:
+ sliced = [table.slice(0, length) for table in row_block]
+ remainder = [table.slice(length, len(row_block[0]) - length) for table in row_block]
+ return sliced, remainder
+
+ def _split_both_like(
+ result: list[list[TableBlock]], blocks: list[list[TableBlock]]
+ ) -> tuple[list[list[TableBlock]], list[list[TableBlock]]]:
+ """
+ Make sure each row_block contain the same num_rows to be able to concatenate them on axis=1.
+
+ To do so, we modify both blocks sets to have the same row_blocks boundaries.
+ For example, if `result` has 2 row_blocks of 3 rows and `blocks` has 3 row_blocks of 2 rows,
+ we modify both to have 4 row_blocks of size 2, 1, 1 and 2:
+
+ [ x x x | x x x ]
+ + [ y y | y y | y y ]
+ -----------------------------
+ = [ x x | x | x | x x ]
+ [ y y | y | y | y y ]
+
+ """
+ result, blocks = list(result), list(blocks)
+ new_result, new_blocks = [], []
+ while result and blocks:
+ # we slice the longest row block to save two row blocks of same length
+ # and we replace the long row block by its remainder if necessary
+ if len(result[0][0]) > len(blocks[0][0]):
+ new_blocks.append(blocks[0])
+ sliced, result[0] = _slice_row_block(result[0], len(blocks.pop(0)[0]))
+ new_result.append(sliced)
+ elif len(result[0][0]) < len(blocks[0][0]):
+ new_result.append(result[0])
+ sliced, blocks[0] = _slice_row_block(blocks[0], len(result.pop(0)[0]))
+ new_blocks.append(sliced)
+ else:
+ new_result.append(result.pop(0))
+ new_blocks.append(blocks.pop(0))
+ if result or blocks:
+ raise ValueError("Failed to concatenate on axis=1 because tables don't have the same number of rows")
+ return new_result, new_blocks
+
+ def _extend_blocks(
+ result: list[list[TableBlock]], blocks: list[list[TableBlock]], axis: int = 0
+ ) -> list[list[TableBlock]]:
+ if axis == 0:
+ result.extend(blocks)
+ elif axis == 1:
+ # We make sure each row_block have the same num_rows
+ result, blocks = _split_both_like(result, blocks)
+ for i, row_block in enumerate(blocks):
+ result[i].extend(row_block)
+ return result
+
+ blocks = to_blocks(tables[0])
+ for table in tables[1:]:
+ table_blocks = to_blocks(table)
+ blocks = _extend_blocks(blocks, table_blocks, axis=axis)
+ return cls.from_blocks(blocks)
+
+ @property
+ def _slices(self):
+ offset = 0
+ for tables in self.blocks:
+ length = len(tables[0])
+ yield (offset, length)
+ offset += length
+
+ def slice(self, offset=0, length=None):
+ """
+ Compute zero-copy slice of this Table.
+
+ Args:
+ offset (`int`, defaults to `0`):
+ Offset from start of table to slice.
+ length (`int`, defaults to `None`):
+ Length of slice (default is until end of table starting from
+ offset).
+
+ Returns:
+ `datasets.table.Table`
+ """
+ table = self.table.slice(offset, length=length)
+ length = length if length is not None else self.num_rows - offset
+ blocks = []
+ for tables in self.blocks:
+ n_rows = len(tables[0])
+ if length == 0:
+ break
+ elif n_rows <= offset:
+ offset = offset - n_rows
+ elif n_rows <= offset + length:
+ blocks.append([t.slice(offset) for t in tables])
+ length, offset = length + offset - n_rows, 0
+ else:
+ blocks.append([t.slice(offset, length) for t in tables])
+ length, offset = 0, 0
+ return ConcatenationTable(table, blocks)
+
+ def filter(self, mask, *args, **kwargs):
+ """
+ Select records from a Table. See `pyarrow.compute.filter` for full usage.
+ """
+ table = self.table.filter(mask, *args, **kwargs)
+ blocks = []
+ for (offset, length), tables in zip(self._slices, self.blocks):
+ submask = mask.slice(offset, length)
+ blocks.append([t.filter(submask, *args, **kwargs) for t in tables])
+ return ConcatenationTable(table, blocks)
+
+ def flatten(self, *args, **kwargs):
+ """
+ Flatten this Table. Each column with a struct type is flattened
+ into one column per struct field. Other columns are left unchanged.
+
+ Args:
+ memory_pool (`MemoryPool`, defaults to `None`):
+ For memory allocations, if required, otherwise use default pool.
+
+ Returns:
+ `datasets.table.Table`
+ """
+ table = table_flatten(self.table, *args, **kwargs)
+ blocks = []
+ for tables in self.blocks:
+ blocks.append([t.flatten(*args, **kwargs) for t in tables])
+ return ConcatenationTable(table, blocks)
+
+ def combine_chunks(self, *args, **kwargs):
+ """
+ Make a new table by combining the chunks this table has.
+
+ All the underlying chunks in the `ChunkedArray` of each column are
+ concatenated into zero or one chunk.
+
+ Args:
+ memory_pool (`MemoryPool`, defaults to `None`):
+ For memory allocations, if required, otherwise use default pool.
+
+ Returns:
+ `datasets.table.Table`
+ """
+ table = self.table.combine_chunks(*args, **kwargs)
+ blocks = []
+ for tables in self.blocks:
+ blocks.append([t.combine_chunks(*args, **kwargs) for t in tables])
+ return ConcatenationTable(table, blocks)
+
+ def cast(self, target_schema, *args, **kwargs):
+ """
+ Cast table values to another schema.
+
+ Args:
+ target_schema (`Schema`):
+ Schema to cast to, the names and order of fields must match.
+ safe (`bool`, defaults to `True`):
+ Check for overflows or other unsafe conversions.
+
+ Returns:
+ `datasets.table.Table`
+ """
+ from .features import Features
+
+ table = table_cast(self.table, target_schema, *args, **kwargs)
+ target_features = Features.from_arrow_schema(target_schema)
+ blocks = []
+ for subtables in self.blocks:
+ new_tables = []
+ fields = list(target_schema)
+ for subtable in subtables:
+ subfields = []
+ for name in subtable.column_names:
+ subfields.append(fields.pop(next(i for i, field in enumerate(fields) if field.name == name)))
+ subfeatures = Features({subfield.name: target_features[subfield.name] for subfield in subfields})
+ subschema = subfeatures.arrow_schema
+ new_tables.append(subtable.cast(subschema, *args, **kwargs))
+ blocks.append(new_tables)
+ return ConcatenationTable(table, blocks)
+
+ def replace_schema_metadata(self, *args, **kwargs):
+ """
+ EXPERIMENTAL: Create shallow copy of table by replacing schema
+ key-value metadata with the indicated new metadata (which may be `None`,
+ which deletes any existing metadata).
+
+ Args:
+ metadata (`dict`, defaults to `None`):
+
+ Returns:
+ `datasets.table.Table`: shallow_copy
+ """
+ table = self.table.replace_schema_metadata(*args, **kwargs)
+ blocks = []
+ for tables in self.blocks:
+ blocks.append([t.replace_schema_metadata(*args, **kwargs) for t in tables])
+ return ConcatenationTable(table, self.blocks)
+
+ def add_column(self, *args, **kwargs):
+ """
+ Add column to Table at position.
+
+ A new table is returned with the column added, the original table
+ object is left unchanged.
+
+ Args:
+ i (`int`):
+ Index to place the column at.
+ field_ (`Union[str, pyarrow.Field]`):
+ If a string is passed then the type is deduced from the column
+ data.
+ column (`Union[pyarrow.Array, List[pyarrow.Array]]`):
+ Column data.
+
+ Returns:
+ `datasets.table.Table`: New table with the passed column added.
+ """
+ raise NotImplementedError()
+
+ def append_column(self, *args, **kwargs):
+ """
+ Append column at end of columns.
+
+ Args:
+ field_ (`Union[str, pyarrow.Field]`):
+ If a string is passed then the type is deduced from the column
+ data.
+ column (`Union[pyarrow.Array, List[pyarrow.Array]]`):
+ Column data.
+
+ Returns:
+ `datasets.table.Table`:
+ New table with the passed column added.
+ """
+ raise NotImplementedError()
+
+ def remove_column(self, i, *args, **kwargs):
+ """
+ Create new Table with the indicated column removed.
+
+ Args:
+ i (`int`):
+ Index of column to remove.
+
+ Returns:
+ `datasets.table.Table`:
+ New table without the column.
+ """
+ table = self.table.remove_column(i, *args, **kwargs)
+ name = self.table.column_names[i]
+ blocks = []
+ for tables in self.blocks:
+ blocks.append(
+ [
+ t.remove_column(t.column_names.index(name), *args, **kwargs) if name in t.column_names else t
+ for t in tables
+ ]
+ )
+ return ConcatenationTable(table, blocks)
+
+ def set_column(self, *args, **kwargs):
+ """
+ Replace column in Table at position.
+
+ Args:
+ i (`int`):
+ Index to place the column at.
+ field_ (`Union[str, pyarrow.Field]`):
+ If a string is passed then the type is deduced from the column
+ data.
+ column (`Union[pyarrow.Array, List[pyarrow.Array]]`):
+ Column data.
+
+ Returns:
+ `datasets.table.Table`:
+ New table with the passed column set.
+ """
+ raise NotImplementedError()
+
+ def rename_columns(self, names, *args, **kwargs):
+ """
+ Create new table with columns renamed to provided names.
+ """
+ table = self.table.rename_columns(names, *args, **kwargs)
+ names = dict(zip(self.table.column_names, names))
+ blocks = []
+ for tables in self.blocks:
+ blocks.append(
+ [t.rename_columns([names[name] for name in t.column_names], *args, **kwargs) for t in tables]
+ )
+ return ConcatenationTable(table, blocks)
+
+ def drop(self, columns, *args, **kwargs):
+ """
+ Drop one or more columns and return a new table.
+
+ Args:
+ columns (`List[str]`):
+ List of field names referencing existing columns.
+
+ Raises:
+ `KeyError` : if any of the passed columns name are not existing.
+
+ Returns:
+ `datasets.table.Table`:
+ New table without the columns.
+ """
+ table = self.table.drop(columns, *args, **kwargs)
+ blocks = []
+ for tables in self.blocks:
+ blocks.append([t.drop([c for c in columns if c in t.column_names], *args, **kwargs) for t in tables])
+ return ConcatenationTable(table, blocks)
+
+ def select(self, columns, *args, **kwargs):
+ """
+ Select columns of the table.
+
+ Returns a new table with the specified columns, and metadata preserved.
+
+ Args:
+ columns (:obj:`Union[List[str], List[int]]`):
+ The column names or integer indices to select.
+
+ Returns:
+ :class:`datasets.table.Table`: New table with the specified columns, and metadata preserved.
+ """
+ table = self.table.select(columns, *args, **kwargs)
+ blocks = []
+ for tables in self.blocks:
+ blocks.append([t.select([c for c in columns if c in t.column_names], *args, **kwargs) for t in tables])
+ return ConcatenationTable(table, blocks)
+
+
+def concat_tables(tables: list[Table], axis: int = 0) -> Table:
+ """
+ Concatenate tables.
+
+ Args:
+ tables (list of `Table`):
+ List of tables to be concatenated.
+ axis (`{0, 1}`, defaults to `0`, meaning over rows):
+ Axis to concatenate over, where `0` means over rows (vertically) and `1` means over columns
+ (horizontally).
+
+
+ Returns:
+ `datasets.table.Table`:
+ If the number of input tables is > 1, then the returned table is a `datasets.table.ConcatenationTable`.
+ Otherwise if there's only one table, it is returned as is.
+ """
+ tables = list(tables)
+ if len(tables) == 1:
+ return tables[0]
+ return ConcatenationTable.from_tables(tables, axis=axis)
+
+
+def list_table_cache_files(table: Table) -> list[str]:
+ """
+ Get the cache files that are loaded by the table.
+ Cache file are used when parts of the table come from the disk via memory mapping.
+
+ Returns:
+ `List[str]`:
+ A list of paths to the cache files loaded by the table.
+ """
+ if isinstance(table, ConcatenationTable):
+ cache_files = []
+ for subtables in table.blocks:
+ for subtable in subtables:
+ cache_files += list_table_cache_files(subtable)
+ return cache_files
+ elif isinstance(table, MemoryMappedTable):
+ return [table.path]
+ else:
+ return []
+
+
+def _wrap_for_chunked_arrays(func):
+ """Apply the function on each chunk of a `pyarrow.ChunkedArray`, or on the array directly"""
+
+ def wrapper(array, *args, **kwargs):
+ if isinstance(array, pa.ChunkedArray):
+ return pa.chunked_array([func(chunk, *args, **kwargs) for chunk in array.chunks])
+ else:
+ return func(array, *args, **kwargs)
+
+ return wrapper
+
+
+def _are_list_values_of_length(array: pa.ListArray, length: int) -> bool:
+ """Check if all the sub-lists of a `pa.ListArray` have the specified length."""
+ return pc.all(pc.equal(array.value_lengths(), length)).as_py() or array.null_count == len(array)
+
+
+def _combine_list_array_offsets_with_mask(array: pa.ListArray) -> pa.Array:
+ """Add the null bitmap to the offsets of a `pa.ListArray`."""
+ offsets = array.offsets
+ if array.null_count > 0:
+ offsets = pa.concat_arrays(
+ [
+ pc.replace_with_mask(offsets[:-1], array.is_null(), pa.nulls(len(array), pa.int32())),
+ offsets[-1:],
+ ]
+ )
+ return offsets
+
+
+def _storage_type(type: pa.DataType) -> pa.DataType:
+ """Convert a (possibly nested) `pa.ExtensionType` to its storage type."""
+ if isinstance(type, pa.ExtensionType):
+ return _storage_type(type.storage_type)
+ elif isinstance(type, pa.StructType):
+ return pa.struct([pa.field(field.name, _storage_type(field.type)) for field in type])
+ elif isinstance(type, pa.ListType):
+ return pa.list_(_storage_type(type.value_type))
+ elif isinstance(type, pa.FixedSizeListType):
+ return pa.list_(_storage_type(type.value_type), type.list_size)
+ return type
+
+
+def _short_str(value: Any) -> str:
+ out = str(value)
+ if len(out) > 3000:
+ out = out[:1500] + "\n...\n" + out[-1500:]
+ return out
+
+
+@_wrap_for_chunked_arrays
+def array_cast(
+ array: pa.Array, pa_type: pa.DataType, allow_primitive_to_str: bool = True, allow_decimal_to_str: bool = True
+) -> Union[pa.Array, pa.FixedSizeListArray, pa.ListArray, pa.StructArray, pa.ExtensionArray]:
+ """Improved version of `pa.Array.cast`
+
+ It supports casting `pa.StructArray` objects to re-order the fields.
+ It also let you control certain aspects of the casting, e.g. whether
+ to disable casting primitives (`booleans`, `floats` or `ints`) or
+ disable casting decimals to strings.
+
+ Args:
+ array (`pa.Array`):
+ PyArrow array to cast
+ pa_type (`pa.DataType`):
+ Target PyArrow type
+ allow_primitive_to_str (`bool`, defaults to `True`):
+ Whether to allow casting primitives to strings.
+ Defaults to `True`.
+ allow_decimal_to_str (`bool`, defaults to `True`):
+ Whether to allow casting decimals to strings.
+ Defaults to `True`.
+
+ Raises:
+ `pa.ArrowInvalidError`: if the arrow data casting fails
+ `TypeError`: if the target type is not supported according, e.g.
+
+ - if a field is missing
+ - if casting from primitives to strings and `allow_primitive_to_str` is `False`
+ - if casting from decimals to strings and `allow_decimal_to_str` is `False`
+
+ Returns:
+ `List[pyarrow.Array]`: the casted array
+ """
+ _c = partial(array_cast, allow_primitive_to_str=allow_primitive_to_str, allow_decimal_to_str=allow_decimal_to_str)
+ if isinstance(array, pa.ExtensionArray):
+ array = array.storage
+ if isinstance(pa_type, pa.ExtensionType):
+ return pa_type.wrap_array(_c(array, pa_type.storage_type))
+ elif array.type == pa_type:
+ return array
+ elif pa.types.is_struct(array.type):
+ if pa.types.is_struct(pa_type) and ({field.name for field in pa_type} == {field.name for field in array.type}):
+ if array.type.num_fields == 0:
+ return array
+ arrays = [_c(array.field(field.name), field.type) for field in pa_type]
+ return pa.StructArray.from_arrays(arrays, fields=list(pa_type), mask=array.is_null())
+ elif pa.types.is_list(array.type) or pa.types.is_large_list(array.type):
+ if pa.types.is_fixed_size_list(pa_type):
+ if _are_list_values_of_length(array, pa_type.list_size):
+ if array.null_count > 0:
+ # Ensure each null value in the array translates to [null] * pa_type.list_size in the array's values array
+ array_type = array.type
+ storage_type = _storage_type(array_type)
+ if array_type != storage_type:
+ # Temporarily convert to the storage type to support extension types in the slice operation
+ array = _c(array, storage_type)
+ array = pc.list_slice(array, 0, pa_type.list_size, return_fixed_size_list=True)
+ array = _c(array, array_type)
+ else:
+ array = pc.list_slice(array, 0, pa_type.list_size, return_fixed_size_list=True)
+ array_values = array.values
+ return pa.FixedSizeListArray.from_arrays(
+ _c(array_values, pa_type.value_type), pa_type.list_size, mask=array.is_null()
+ )
+ else:
+ array_values = array.values[
+ array.offset * pa_type.list_size : (array.offset + len(array)) * pa_type.list_size
+ ]
+ return pa.FixedSizeListArray.from_arrays(_c(array_values, pa_type.value_type), pa_type.list_size)
+ elif pa.types.is_list(pa_type):
+ # Merge offsets with the null bitmap to avoid the "Null bitmap with offsets slice not supported" ArrowNotImplementedError
+ array_offsets = _combine_list_array_offsets_with_mask(array)
+ return pa.ListArray.from_arrays(array_offsets, _c(array.values, pa_type.value_type))
+ elif pa.types.is_large_list(pa_type):
+ # Merge offsets with the null bitmap to avoid the "Null bitmap with offsets slice not supported" ArrowNotImplementedError
+ array_offsets = _combine_list_array_offsets_with_mask(array)
+ return pa.LargeListArray.from_arrays(array_offsets, _c(array.values, pa_type.value_type))
+ elif pa.types.is_fixed_size_list(array.type):
+ if pa.types.is_fixed_size_list(pa_type):
+ if pa_type.list_size == array.type.list_size:
+ array_values = array.values[
+ array.offset * array.type.list_size : (array.offset + len(array)) * array.type.list_size
+ ]
+ return pa.FixedSizeListArray.from_arrays(
+ _c(array_values, pa_type.value_type), pa_type.list_size, mask=array.is_null()
+ )
+ elif pa.types.is_list(pa_type):
+ array_offsets = (np.arange(len(array) + 1) + array.offset) * array.type.list_size
+ return pa.ListArray.from_arrays(array_offsets, _c(array.values, pa_type.value_type), mask=array.is_null())
+ elif pa.types.is_large_list(pa_type):
+ array_offsets = (np.arange(len(array) + 1) + array.offset) * array.type.list_size
+ return pa.LargeListArray.from_arrays(
+ array_offsets, _c(array.values, pa_type.value_type), mask=array.is_null()
+ )
+ else:
+ if pa.types.is_string(pa_type):
+ if not allow_primitive_to_str and pa.types.is_primitive(array.type):
+ raise TypeError(
+ f"Couldn't cast array of type {_short_str(array.type)} to {_short_str(pa_type)} "
+ f"since allow_primitive_to_str is set to {allow_primitive_to_str} "
+ )
+ if not allow_decimal_to_str and pa.types.is_decimal(array.type):
+ raise TypeError(
+ f"Couldn't cast array of type {_short_str(array.type)} to {_short_str(pa_type)} "
+ f"and allow_decimal_to_str is set to {allow_decimal_to_str}"
+ )
+ if pa.types.is_null(pa_type) and not pa.types.is_null(array.type):
+ raise TypeError(f"Couldn't cast array of type {_short_str(array.type)} to {_short_str(pa_type)}")
+ return array.cast(pa_type)
+ raise TypeError(f"Couldn't cast array of type {_short_str(array.type)} to {_short_str(pa_type)}")
+
+
+@_wrap_for_chunked_arrays
+def cast_array_to_feature(
+ array: pa.Array, feature: "FeatureType", allow_primitive_to_str: bool = True, allow_decimal_to_str: bool = True
+) -> pa.Array:
+ """Cast an array to the arrow type that corresponds to the requested feature type.
+ For custom features like [`Audio`] or [`Image`], it takes into account the "cast_storage" methods
+ they defined to enable casting from other arrow types.
+
+ Args:
+ array (`pa.Array`):
+ The PyArrow array to cast.
+ feature (`datasets.features.FeatureType`):
+ The target feature type.
+ allow_primitive_to_str (`bool`, defaults to `True`):
+ Whether to allow casting primitives to strings.
+ Defaults to `True`.
+ allow_decimal_to_str (`bool`, defaults to `True`):
+ Whether to allow casting decimals to strings.
+ Defaults to `True`.
+
+ Raises:
+ `pa.ArrowInvalidError`: if the arrow data casting fails
+ `TypeError`: if the target type is not supported according, e.g.
+
+ - if a field is missing
+ - if casting from primitives and `allow_primitive_to_str` is `False`
+ - if casting from decimals and `allow_decimal_to_str` is `False`
+
+ Returns:
+ array (`pyarrow.Array`): the casted array
+ """
+ from .features.features import LargeList, List, get_nested_type
+
+ _c = partial(
+ cast_array_to_feature,
+ allow_primitive_to_str=allow_primitive_to_str,
+ allow_decimal_to_str=allow_decimal_to_str,
+ )
+
+ if isinstance(array, pa.ExtensionArray):
+ array = array.storage
+ if hasattr(feature, "cast_storage"):
+ return feature.cast_storage(array)
+
+ if pa.types.is_struct(array.type):
+ # feature must be a dict
+ if isinstance(feature, dict) and (array_fields := {field.name for field in array.type}) <= set(feature):
+ null_array = pa.array([None] * len(array))
+ arrays = [
+ _c(array.field(name) if name in array_fields else null_array, subfeature)
+ for name, subfeature in feature.items()
+ ]
+ return pa.StructArray.from_arrays(arrays, names=list(feature), mask=array.is_null())
+ elif pa.types.is_list(array.type) or pa.types.is_large_list(array.type):
+ # feature must be either List(subfeature) or LargeList(subfeature)
+ if isinstance(feature, LargeList):
+ casted_array_values = _c(array.values, feature.feature)
+ if pa.types.is_large_list(array.type) and casted_array_values.type == array.values.type:
+ # Both array and feature have equal large_list type and values (within the list) type
+ return array
+ else:
+ # Merge offsets with the null bitmap to avoid the "Null bitmap with offsets slice not supported" ArrowNotImplementedError
+ array_offsets = _combine_list_array_offsets_with_mask(array)
+ return pa.LargeListArray.from_arrays(array_offsets, casted_array_values)
+ elif isinstance(feature, List):
+ if feature.length > -1:
+ if _are_list_values_of_length(array, feature.length):
+ if array.null_count > 0:
+ # Ensure each null value in the array translates to [null] * pa_type.list_size in the array's values array
+ array_type = array.type
+ storage_type = _storage_type(array_type)
+ if array_type != storage_type:
+ # Temporarily convert to the storage type to support extension types in the slice operation
+ array = array_cast(
+ array,
+ storage_type,
+ allow_primitive_to_str=allow_primitive_to_str,
+ allow_decimal_to_str=allow_decimal_to_str,
+ )
+ array = pc.list_slice(array, 0, feature.length, return_fixed_size_list=True)
+ array = array_cast(
+ array,
+ array_type,
+ allow_primitive_to_str=allow_primitive_to_str,
+ allow_decimal_to_str=allow_decimal_to_str,
+ )
+ else:
+ array = pc.list_slice(array, 0, feature.length, return_fixed_size_list=True)
+ array_values = array.values
+ casted_array_values = _c(array_values, feature.feature)
+ return pa.FixedSizeListArray.from_arrays(
+ casted_array_values, feature.length, mask=array.is_null()
+ )
+ else:
+ array_values = array.values[
+ array.offset * feature.length : (array.offset + len(array)) * feature.length
+ ]
+ return pa.FixedSizeListArray.from_arrays(_c(array_values, feature.feature), feature.length)
+ else:
+ casted_array_values = _c(array.values, feature.feature)
+ if pa.types.is_list(array.type) and casted_array_values.type == array.values.type:
+ # Both array and feature have equal list type and values (within the list) type
+ return array
+ else:
+ # Merge offsets with the null bitmap to avoid the "Null bitmap with offsets slice not supported" ArrowNotImplementedError
+ array_offsets = _combine_list_array_offsets_with_mask(array)
+ return pa.ListArray.from_arrays(array_offsets, casted_array_values)
+ elif pa.types.is_fixed_size_list(array.type):
+ # feature must be List(subfeature)
+ if isinstance(feature, LargeList):
+ array_offsets = (np.arange(len(array) + 1) + array.offset) * array.type.list_size
+ return pa.LargeListArray.from_arrays(
+ array_offsets, _c(array.values, feature.feature), mask=array.is_null()
+ )
+ elif isinstance(feature, List):
+ if feature.length > -1:
+ if feature.length == array.type.list_size:
+ array_values = array.values[
+ array.offset * array.type.list_size : (array.offset + len(array)) * array.type.list_size
+ ]
+ casted_array_values = _c(array_values, feature.feature)
+ return pa.FixedSizeListArray.from_arrays(casted_array_values, feature.length, mask=array.is_null())
+ else:
+ array_offsets = (np.arange(len(array) + 1) + array.offset) * array.type.list_size
+ return pa.ListArray.from_arrays(array_offsets, _c(array.values, feature.feature), mask=array.is_null())
+ if pa.types.is_null(array.type):
+ return array_cast(
+ array,
+ get_nested_type(feature),
+ allow_primitive_to_str=allow_primitive_to_str,
+ allow_decimal_to_str=allow_decimal_to_str,
+ )
+ elif not isinstance(feature, (List, LargeList, dict)):
+ return array_cast(
+ array,
+ feature(),
+ allow_primitive_to_str=allow_primitive_to_str,
+ allow_decimal_to_str=allow_decimal_to_str,
+ )
+ raise TypeError(f"Couldn't cast array of type\n{_short_str(array.type)}\nto\n{_short_str(feature)}")
+
+
+@_wrap_for_chunked_arrays
+def embed_array_storage(
+ array: pa.Array,
+ feature: "FeatureType",
+ token_per_repo_id=None,
+ local_files: bool = True,
+ remote_files: bool = True,
+):
+ """Embed data into an arrays's storage.
+ For custom features like Audio or Image, it takes into account the "embed_storage" methods
+ they define to embed external data (e.g. an image file) into an array.
+
+
+
+ Args:
+ array (`pa.Array`):
+ The PyArrow array in which to embed data.
+ feature (`datasets.features.FeatureType`):
+ Array features.
+ local_files (`bool`, defaults to `True`)
+ Whether to embed local files data in the array
+
+
+ remote_files (`bool`, defaults to `True`)
+ Whether to embed remote files data in the array.
+ E.g. files with paths that start with hf:// or https://
+
+
+
+ Raises:
+ `TypeError`: if the target type is not supported according, e.g.
+
+ - if a field is missing
+
+ Returns:
+ array (`pyarrow.Array`): the casted array
+ """
+ if not local_files and not remote_files:
+ return array
+
+ from .features import LargeList, List
+
+ _e = partial(
+ embed_array_storage, token_per_repo_id=token_per_repo_id, local_files=local_files, remote_files=remote_files
+ )
+
+ if isinstance(array, pa.ExtensionArray):
+ array = array.storage
+ if hasattr(feature, "embed_storage"):
+ return feature.embed_storage(
+ array, token_per_repo_id=token_per_repo_id, local_files=local_files, remote_files=remote_files
+ )
+ elif pa.types.is_struct(array.type):
+ # feature must be a dict
+ if isinstance(feature, dict):
+ arrays = [_e(array.field(name), subfeature) for name, subfeature in feature.items()]
+ return pa.StructArray.from_arrays(arrays, names=list(feature), mask=array.is_null())
+ elif pa.types.is_list(array.type):
+ # feature must be either List(subfeature)
+ # Merge offsets with the null bitmap to avoid the "Null bitmap with offsets slice not supported" ArrowNotImplementedError
+ array_offsets = _combine_list_array_offsets_with_mask(array)
+ if isinstance(feature, List) and feature.length == -1:
+ return pa.ListArray.from_arrays(array_offsets, _e(array.values, feature.feature))
+ elif pa.types.is_large_list(array.type):
+ # feature must be LargeList(subfeature)
+ # Merge offsets with the null bitmap to avoid the "Null bitmap with offsets slice not supported" ArrowNotImplementedError
+ array_offsets = _combine_list_array_offsets_with_mask(array)
+ return pa.LargeListArray.from_arrays(array_offsets, _e(array.values, feature.feature))
+ elif pa.types.is_fixed_size_list(array.type):
+ # feature must be List(subfeature)
+ if isinstance(feature, List) and feature.length > -1:
+ array_values = array.values[
+ array.offset * array.type.list_size : (array.offset + len(array)) * array.type.list_size
+ ]
+ embedded_array_values = _e(array_values, feature.feature)
+ return pa.FixedSizeListArray.from_arrays(embedded_array_values, feature.length, mask=array.is_null())
+ if not isinstance(feature, (List, LargeList, dict)):
+ return array
+ raise TypeError(f"Couldn't embed array of type\n{_short_str(array.type)}\nwith\n{_short_str(feature)}")
+
+
+class CastError(ValueError):
+ """When it's not possible to cast an Arrow table to a specific schema or set of features"""
+
+ def __init__(self, *args, table_column_names: list[str], requested_column_names: list[str]) -> None:
+ super().__init__(*args)
+ self.table_column_names = table_column_names
+ self.requested_column_names = requested_column_names
+
+ def __reduce__(self):
+ # Fix unpickling: TypeError: __init__() missing 2 required keyword-only arguments: 'table_column_names' and 'requested_column_names'
+ return partial(
+ CastError, table_column_names=self.table_column_names, requested_column_names=self.requested_column_names
+ ), ()
+
+ def details(self):
+ new_columns = set(self.table_column_names) - set(self.requested_column_names)
+ missing_columns = set(self.requested_column_names) - set(self.table_column_names)
+ if new_columns and missing_columns:
+ return f"there are {len(new_columns)} new columns ({_short_str(new_columns)}) and {len(missing_columns)} missing columns ({_short_str(missing_columns)})."
+ elif new_columns:
+ return f"there are {len(new_columns)} new columns ({_short_str(new_columns)})"
+ else:
+ return f"there are {len(missing_columns)} missing columns ({_short_str(missing_columns)})"
+
+
+def cast_table_to_features(table: pa.Table, features: "Features"):
+ """Cast a table to the arrow schema that corresponds to the requested features.
+
+ Args:
+ table (`pyarrow.Table`):
+ PyArrow table to cast.
+ features ([`Features`]):
+ Target features.
+
+ Returns:
+ table (`pyarrow.Table`): the casted table
+ """
+ if sorted(table.column_names) != sorted(features):
+ raise CastError(
+ f"Couldn't cast\n{_short_str(table.schema)}\nto\n{_short_str(features)}\nbecause column names don't match",
+ table_column_names=table.column_names,
+ requested_column_names=list(features),
+ )
+ arrays = [cast_array_to_feature(table[name], feature) for name, feature in features.items()]
+ return pa.Table.from_arrays(arrays, schema=features.arrow_schema)
+
+
+def cast_table_to_schema(table: pa.Table, schema: pa.Schema):
+ """Cast a table to the arrow schema. Different from `cast_table_to_features`, this method can preserve nullability.
+
+ Args:
+ table (`pa.Table`):
+ PyArrow table to cast.
+ features ([`Features`]):
+ Target features.
+
+ Returns:
+ `pa.Table`: the casted table
+ """
+ from .features import Features
+
+ features = Features.from_arrow_schema(schema)
+ table_column_names = set(table.column_names)
+ if not table_column_names <= set(schema.names):
+ raise CastError(
+ f"Couldn't cast\n{_short_str(table.schema)}\nto\n{_short_str(features)}\nbecause column names don't match",
+ table_column_names=table.column_names,
+ requested_column_names=list(features),
+ )
+ arrays = [
+ cast_array_to_feature(
+ table[name] if name in table_column_names else pa.array([None] * len(table), type=schema.field(name).type),
+ feature,
+ )
+ for name, feature in features.items()
+ ]
+ return pa.Table.from_arrays(arrays, schema=schema)
+
+
+def embed_table_storage(table: pa.Table, token_per_repo_id=None, local_files: bool = True, remote_files: bool = True):
+ """Embed external data into a table's storage.
+
+
+
+ Args:
+ table (`pyarrow.Table`):
+ PyArrow table in which to embed data.
+ local_files (`bool`, defaults to `True`)
+ Whether to embed local files data in the table
+
+
+ remote_files (`bool`, defaults to `True`)
+ Whether to embed remote files data in the table.
+ E.g. files with paths that start with hf:// or https://
+
+
+
+ Returns:
+ table (`pyarrow.Table`): the table with embedded data
+ """
+ if not local_files and not remote_files:
+ return table
+
+ from .features.features import Features, require_storage_embed
+
+ features = Features.from_arrow_schema(table.schema)
+ arrays = [
+ embed_array_storage(
+ table[name],
+ feature,
+ token_per_repo_id=token_per_repo_id,
+ local_files=local_files,
+ remote_files=remote_files,
+ )
+ if require_storage_embed(feature)
+ else table[name]
+ for name, feature in features.items()
+ ]
+ return pa.Table.from_arrays(arrays, schema=features.arrow_schema)
+
+
+def table_cast(table: pa.Table, schema: pa.Schema):
+ """Improved version of `pa.Table.cast`.
+
+ It supports casting to feature types stored in the schema metadata.
+
+ Args:
+ table (`pyarrow.Table`):
+ PyArrow table to cast.
+ schema (`pyarrow.Schema`):
+ Target PyArrow schema.
+
+ Returns:
+ table (`pyarrow.Table`): the casted table
+ """
+ if table.schema != schema:
+ return cast_table_to_schema(table, schema)
+ elif table.schema.metadata != schema.metadata:
+ return table.replace_schema_metadata(schema.metadata)
+ else:
+ return table
+
+
+def table_flatten(table: pa.Table):
+ """Improved version of `pa.Table.flatten`.
+
+ It behaves as `pa.Table.flatten` in a sense it does 1-step flatten of the columns with a struct type into one column per struct field,
+ but updates the metadata and skips decodable features unless the `decode` attribute of these features is set to False.
+
+ Args:
+ table (`pa.Table`):
+ PyArrow table to flatten.
+
+ Returns:
+ `Table`: the flattened table
+ """
+ from .features import Features
+
+ features = Features.from_arrow_schema(table.schema)
+ if any(hasattr(subfeature, "flatten") and subfeature.flatten() == subfeature for subfeature in features.values()):
+ flat_arrays = []
+ flat_column_names = []
+ for field in table.schema:
+ array = table.column(field.name)
+ subfeature = features[field.name]
+ if pa.types.is_struct(field.type) and (
+ not hasattr(subfeature, "flatten") or subfeature.flatten() != subfeature
+ ):
+ flat_arrays.extend(array.flatten())
+ flat_column_names.extend([f"{field.name}.{subfield.name}" for subfield in field.type])
+ else:
+ flat_arrays.append(array)
+ flat_column_names.append(field.name)
+ flat_table = pa.Table.from_arrays(
+ flat_arrays,
+ names=flat_column_names,
+ )
+ else:
+ flat_table = table.flatten()
+ # Preserve complex types in the metadata
+ flat_features = features.flatten(max_depth=2)
+ flat_features = Features({column_name: flat_features[column_name] for column_name in flat_table.column_names})
+ return flat_table.replace_schema_metadata(flat_features.arrow_schema.metadata)
+
+
+def table_visitor(table: pa.Table, function: Callable[[pa.Array], None]):
+ """Visit all arrays in a table and apply a function to them.
+
+ Args:
+ table (`pyarrow.Table`):
+ PyArrow table to visit.
+ function (`Callable[[pa.Array], None]`):
+ Function to apply to each array.
+ """
+ from .features import Features, LargeList, List
+
+ features = Features.from_arrow_schema(table.schema)
+
+ def _visit(array, feature):
+ if isinstance(array, pa.ChunkedArray):
+ for chunk in array.chunks:
+ _visit(chunk, feature)
+ else:
+ if isinstance(array, pa.ExtensionArray):
+ array = array.storage
+ function(array, feature)
+ if pa.types.is_struct(array.type) and not hasattr(feature, "cast_storage"):
+ for name, subfeature in feature.items():
+ _visit(array.field(name), subfeature)
+ elif pa.types.is_list(array.type):
+ if isinstance(feature, (LargeList, List)):
+ _visit(array.values, feature.feature)
+
+ for name, feature in features.items():
+ _visit(table[name], feature)
+
+
+def table_iter(table: Table, batch_size: int, drop_last_batch=False) -> Iterator[pa.Table]:
+ """Iterate over sub-tables of size `batch_size`.
+
+ Args:
+ table (`pyarrow.Table`):
+ PyArrow table to iterate over.
+ batch_size (`int`):
+ Size of each sub-table to yield.
+ drop_last_batch (`bool`, defaults to `False`):
+ Drop the last batch if it is smaller than `batch_size`.
+ """
+ chunks_buffer = []
+ chunks_buffer_size = 0
+ for chunk in table.to_reader(max_chunksize=batch_size):
+ if len(chunk) == 0:
+ continue
+ elif chunks_buffer_size + len(chunk) < batch_size:
+ chunks_buffer.append(chunk)
+ chunks_buffer_size += len(chunk)
+ continue
+ elif chunks_buffer_size + len(chunk) == batch_size:
+ chunks_buffer.append(chunk)
+ yield pa.Table.from_batches(chunks_buffer)
+ chunks_buffer = []
+ chunks_buffer_size = 0
+ else:
+ cropped_chunk_length = batch_size - chunks_buffer_size
+ chunks_buffer.append(chunk.slice(0, cropped_chunk_length))
+ yield pa.Table.from_batches(chunks_buffer)
+ chunks_buffer = [chunk.slice(cropped_chunk_length, len(chunk) - cropped_chunk_length)]
+ chunks_buffer_size = len(chunk) - cropped_chunk_length
+ if not drop_last_batch and chunks_buffer:
+ yield pa.Table.from_batches(chunks_buffer)
diff --git a/venv/Lib/site-packages/datasets/utils/__init__.py b/venv/Lib/site-packages/datasets/utils/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..a1c3d2935994956e230a441f3f2d65695ebaaf2d
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/__init__.py
@@ -0,0 +1,25 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from . import tqdm as _tqdm # _tqdm is the module
+from .experimental import experimental
+from .info_utils import VerificationMode
+from .logging import disable_progress_bar, enable_progress_bar, is_progress_bar_enabled
+from .tqdm import (
+ are_progress_bars_disabled,
+ disable_progress_bars,
+ enable_progress_bars,
+ tqdm,
+)
+from .version import Version
diff --git a/venv/Lib/site-packages/datasets/utils/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/utils/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6f6fec96ea31d3cca792d97e0a92b83e167864a2
Binary files /dev/null and b/venv/Lib/site-packages/datasets/utils/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/utils/__pycache__/_dataset_viewer.cpython-311.pyc b/venv/Lib/site-packages/datasets/utils/__pycache__/_dataset_viewer.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..fa8518fdbb39cb06c29d3db65d414f8af49d2a89
Binary files /dev/null and b/venv/Lib/site-packages/datasets/utils/__pycache__/_dataset_viewer.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/utils/__pycache__/_dill.cpython-311.pyc b/venv/Lib/site-packages/datasets/utils/__pycache__/_dill.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9ff0374973583d85921e1ca8debecd20d4013878
Binary files /dev/null and b/venv/Lib/site-packages/datasets/utils/__pycache__/_dill.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/utils/__pycache__/_filelock.cpython-311.pyc b/venv/Lib/site-packages/datasets/utils/__pycache__/_filelock.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..41e1ed6dc3ee1efd61eda10aea9a9afaf7521e91
Binary files /dev/null and b/venv/Lib/site-packages/datasets/utils/__pycache__/_filelock.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/utils/__pycache__/deprecation_utils.cpython-311.pyc b/venv/Lib/site-packages/datasets/utils/__pycache__/deprecation_utils.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0d7856a8c9ee105188b9fbcbe6cf0cebc0f1e9b4
Binary files /dev/null and b/venv/Lib/site-packages/datasets/utils/__pycache__/deprecation_utils.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/utils/__pycache__/doc_utils.cpython-311.pyc b/venv/Lib/site-packages/datasets/utils/__pycache__/doc_utils.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8667c0206d2c70a60aa7fb09a275ec4db64ed9b9
Binary files /dev/null and b/venv/Lib/site-packages/datasets/utils/__pycache__/doc_utils.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/utils/__pycache__/experimental.cpython-311.pyc b/venv/Lib/site-packages/datasets/utils/__pycache__/experimental.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..73b93ca08cc43d2e133f9317ff2ca9acd10ea202
Binary files /dev/null and b/venv/Lib/site-packages/datasets/utils/__pycache__/experimental.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/utils/__pycache__/extract.cpython-311.pyc b/venv/Lib/site-packages/datasets/utils/__pycache__/extract.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..00def4da4b48d3418f025aacabc35b708485abae
Binary files /dev/null and b/venv/Lib/site-packages/datasets/utils/__pycache__/extract.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/utils/__pycache__/file_utils.cpython-311.pyc b/venv/Lib/site-packages/datasets/utils/__pycache__/file_utils.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..001c461a38d056c62d2a563d1d673abb158caf70
Binary files /dev/null and b/venv/Lib/site-packages/datasets/utils/__pycache__/file_utils.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/utils/__pycache__/filelock.cpython-311.pyc b/venv/Lib/site-packages/datasets/utils/__pycache__/filelock.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6c8726f85597852f5617491d9ce141dde4b3918b
Binary files /dev/null and b/venv/Lib/site-packages/datasets/utils/__pycache__/filelock.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/utils/__pycache__/hub.cpython-311.pyc b/venv/Lib/site-packages/datasets/utils/__pycache__/hub.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..5a8205fea6ee5b2ff78c343eade0240a4f8962bd
Binary files /dev/null and b/venv/Lib/site-packages/datasets/utils/__pycache__/hub.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/utils/__pycache__/info_utils.cpython-311.pyc b/venv/Lib/site-packages/datasets/utils/__pycache__/info_utils.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3301d3fd8ada31208cb428168afb7d410e2a2ff1
Binary files /dev/null and b/venv/Lib/site-packages/datasets/utils/__pycache__/info_utils.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/utils/__pycache__/json.cpython-311.pyc b/venv/Lib/site-packages/datasets/utils/__pycache__/json.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9719acc0261cbe21e4cbf2499feee86f3cda08a7
Binary files /dev/null and b/venv/Lib/site-packages/datasets/utils/__pycache__/json.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/utils/__pycache__/logging.cpython-311.pyc b/venv/Lib/site-packages/datasets/utils/__pycache__/logging.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6718e67f87b0d303b63013ff39d25ded18f0dbb7
Binary files /dev/null and b/venv/Lib/site-packages/datasets/utils/__pycache__/logging.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/utils/__pycache__/metadata.cpython-311.pyc b/venv/Lib/site-packages/datasets/utils/__pycache__/metadata.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4e9804f9f6f87787f5ad1ed4ab226df3a8265022
Binary files /dev/null and b/venv/Lib/site-packages/datasets/utils/__pycache__/metadata.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/utils/__pycache__/patching.cpython-311.pyc b/venv/Lib/site-packages/datasets/utils/__pycache__/patching.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..08d364e6f05a15e567b4500fc3fd73afb6fa6f5d
Binary files /dev/null and b/venv/Lib/site-packages/datasets/utils/__pycache__/patching.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/utils/__pycache__/py_utils.cpython-311.pyc b/venv/Lib/site-packages/datasets/utils/__pycache__/py_utils.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..35dd0fafb1bc4621bc3b00ddbbe42b10c2b370d6
Binary files /dev/null and b/venv/Lib/site-packages/datasets/utils/__pycache__/py_utils.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/utils/__pycache__/sharding.cpython-311.pyc b/venv/Lib/site-packages/datasets/utils/__pycache__/sharding.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c7ccb32cfd63550699554a88908add81a9e173fb
Binary files /dev/null and b/venv/Lib/site-packages/datasets/utils/__pycache__/sharding.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/utils/__pycache__/stratify.cpython-311.pyc b/venv/Lib/site-packages/datasets/utils/__pycache__/stratify.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ceeef26eb31ef2063310b7ca43db5fe5dcd76185
Binary files /dev/null and b/venv/Lib/site-packages/datasets/utils/__pycache__/stratify.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/utils/__pycache__/tf_utils.cpython-311.pyc b/venv/Lib/site-packages/datasets/utils/__pycache__/tf_utils.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..11586df2b53cd15da12be0ea85df383c354451d1
Binary files /dev/null and b/venv/Lib/site-packages/datasets/utils/__pycache__/tf_utils.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/utils/__pycache__/tqdm.cpython-311.pyc b/venv/Lib/site-packages/datasets/utils/__pycache__/tqdm.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c727dbe63376a6b86d5ac9cbb8357393a38edf67
Binary files /dev/null and b/venv/Lib/site-packages/datasets/utils/__pycache__/tqdm.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/utils/__pycache__/track.cpython-311.pyc b/venv/Lib/site-packages/datasets/utils/__pycache__/track.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7bd73a27c1e202f61783d5ef7314f9aed18c4ac8
Binary files /dev/null and b/venv/Lib/site-packages/datasets/utils/__pycache__/track.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/utils/__pycache__/typing.cpython-311.pyc b/venv/Lib/site-packages/datasets/utils/__pycache__/typing.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0adddb5a5d63ab11c1e7552542e9cfce9e5dba2b
Binary files /dev/null and b/venv/Lib/site-packages/datasets/utils/__pycache__/typing.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/utils/__pycache__/version.cpython-311.pyc b/venv/Lib/site-packages/datasets/utils/__pycache__/version.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..a07ef592b96745275fb11c944bded51efa43fe2f
Binary files /dev/null and b/venv/Lib/site-packages/datasets/utils/__pycache__/version.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/utils/_dataset_viewer.py b/venv/Lib/site-packages/datasets/utils/_dataset_viewer.py
new file mode 100644
index 0000000000000000000000000000000000000000..0f4c298646c5534a93ecfd217b84987b4786cec6
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/_dataset_viewer.py
@@ -0,0 +1,95 @@
+from typing import Any, Optional, Union
+
+from huggingface_hub.utils import get_session
+
+from .. import config
+from ..exceptions import DatasetsError
+from .file_utils import (
+ get_authentication_headers_for_url,
+)
+from .logging import get_logger
+
+
+logger = get_logger(__name__)
+
+
+class DatasetViewerError(DatasetsError):
+ """Dataset viewer error.
+
+ Raised when trying to use the dataset viewer HTTP API and when trying to access:
+ - a missing dataset, or
+ - a private/gated dataset and the user is not authenticated.
+ - unavailable /parquet or /info responses
+ """
+
+
+def get_exported_parquet_files(
+ dataset: str, commit_hash: str, token: Optional[Union[str, bool]]
+) -> list[dict[str, Any]]:
+ """
+ Get the dataset exported parquet files
+ Docs: https://huggingface.co/docs/datasets-server/parquet
+ """
+ dataset_viewer_parquet_url = config.HF_ENDPOINT.replace("://", "://datasets-server.") + "/parquet?dataset="
+ try:
+ parquet_data_files_response = get_session().get(
+ url=dataset_viewer_parquet_url + dataset,
+ headers=get_authentication_headers_for_url(config.HF_ENDPOINT + f"datasets/{dataset}", token=token),
+ timeout=100.0,
+ )
+ parquet_data_files_response.raise_for_status()
+ if "X-Revision" in parquet_data_files_response.headers:
+ if parquet_data_files_response.headers["X-Revision"] == commit_hash or commit_hash is None:
+ parquet_data_files_response_json = parquet_data_files_response.json()
+ if (
+ parquet_data_files_response_json.get("partial") is False
+ and not parquet_data_files_response_json.get("pending", True)
+ and not parquet_data_files_response_json.get("failed", True)
+ and "parquet_files" in parquet_data_files_response_json
+ ):
+ return parquet_data_files_response_json["parquet_files"]
+ else:
+ logger.debug(f"Parquet export for {dataset} is not completely ready yet.")
+ else:
+ logger.debug(
+ f"Parquet export for {dataset} is available but outdated (commit_hash='{parquet_data_files_response.headers['X-Revision']}')"
+ )
+ except Exception as e: # noqa catch any exception of the dataset viewer API and consider the parquet export doesn't exist
+ logger.debug(f"No parquet export for {dataset} available ({type(e).__name__}: {e})")
+ raise DatasetViewerError("No exported Parquet files available.")
+
+
+def get_exported_dataset_infos(
+ dataset: str, commit_hash: str, token: Optional[Union[str, bool]]
+) -> dict[str, dict[str, Any]]:
+ """
+ Get the dataset information, can be useful to get e.g. the dataset features.
+ Docs: https://huggingface.co/docs/datasets-server/info
+ """
+ dataset_viewer_info_url = config.HF_ENDPOINT.replace("://", "://datasets-server.") + "/info?dataset="
+ try:
+ info_response = get_session().get(
+ url=dataset_viewer_info_url + dataset,
+ headers=get_authentication_headers_for_url(config.HF_ENDPOINT + f"datasets/{dataset}", token=token),
+ timeout=100.0,
+ )
+ info_response.raise_for_status()
+ if "X-Revision" in info_response.headers:
+ if info_response.headers["X-Revision"] == commit_hash or commit_hash is None:
+ info_response = info_response.json()
+ if (
+ info_response.get("partial") is False
+ and not info_response.get("pending", True)
+ and not info_response.get("failed", True)
+ and "dataset_info" in info_response
+ ):
+ return info_response["dataset_info"]
+ else:
+ logger.debug(f"Dataset info for {dataset} is not completely ready yet.")
+ else:
+ logger.debug(
+ f"Dataset info for {dataset} is available but outdated (commit_hash='{info_response.headers['X-Revision']}')"
+ )
+ except Exception as e: # noqa catch any exception of the dataset viewer API and consider the dataset info doesn't exist
+ logger.debug(f"No dataset info for {dataset} available ({type(e).__name__}: {e})")
+ raise DatasetViewerError("No exported dataset infos available.")
diff --git a/venv/Lib/site-packages/datasets/utils/_dill.py b/venv/Lib/site-packages/datasets/utils/_dill.py
new file mode 100644
index 0000000000000000000000000000000000000000..18300966b089483c692e84f9aa854b34d6a6958a
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/_dill.py
@@ -0,0 +1,505 @@
+# Copyright 2023 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Extends `dill` to support pickling more types and produce more consistent dumps."""
+
+import os
+import sys
+from io import BytesIO
+from types import CodeType, FunctionType
+
+import dill
+from packaging import version
+
+from .. import config
+
+
+class Pickler(dill.Pickler):
+ dispatch = dill._dill.MetaCatchingDict(dill.Pickler.dispatch.copy())
+ _legacy_no_dict_keys_sorting = False
+
+ def save(self, obj, save_persistent_id=True):
+ obj_type = type(obj)
+ if obj_type not in self.dispatch:
+ if "regex" in sys.modules:
+ import regex # type: ignore
+
+ if obj_type is regex.Pattern:
+ pklregister(obj_type)(_save_regexPattern)
+ if "spacy" in sys.modules:
+ import spacy # type: ignore
+
+ if issubclass(obj_type, spacy.Language):
+ pklregister(obj_type)(_save_spacyLanguage)
+ if "tiktoken" in sys.modules:
+ import tiktoken # type: ignore
+
+ if obj_type is tiktoken.Encoding:
+ pklregister(obj_type)(_save_tiktokenEncoding)
+ if "torch" in sys.modules:
+ import torch # type: ignore
+
+ if issubclass(obj_type, torch.Tensor):
+ pklregister(obj_type)(_save_torchTensor)
+
+ if obj_type is torch.Generator:
+ pklregister(obj_type)(_save_torchGenerator)
+
+ # Unwrap `torch.compile`-ed modules
+ if issubclass(obj_type, torch.nn.Module):
+ obj = getattr(obj, "_orig_mod", obj)
+ if "transformers" in sys.modules:
+ import transformers # type: ignore
+
+ if issubclass(obj_type, transformers.PreTrainedTokenizerBase):
+ pklregister(obj_type)(_save_transformersPreTrainedTokenizerBase)
+
+ # Unwrap `torch.compile`-ed functions
+ if obj_type is FunctionType:
+ obj = getattr(obj, "_torchdynamo_orig_callable", obj)
+ dill.Pickler.save(self, obj, save_persistent_id=save_persistent_id)
+
+ def _batch_setitems(self, items, *args, **kwargs):
+ # Ignore the order of keys in a dict
+ try:
+ # Faster, but fails for unorderable elements
+ items = sorted(items)
+ except Exception: # TypeError, decimal.InvalidOperation, etc.
+ from datasets.fingerprint import Hasher
+
+ items = sorted(items, key=lambda x: Hasher.hash(x[0]))
+ return super()._batch_setitems(items, *args, **kwargs)
+
+ def memoize(self, obj):
+ # Don't memoize strings since two identical strings can have different Python ids
+ if type(obj) is not str: # noqa: E721
+ dill.Pickler.memoize(self, obj)
+
+
+def pklregister(t):
+ """Register a custom reducer for the type."""
+
+ def proxy(func):
+ Pickler.dispatch[t] = func
+ return func
+
+ return proxy
+
+
+def _is_supported_dill_version():
+ """Check if the current dill version is in the supported range."""
+ return config.DILL_VERSION.release[:3] in [
+ version.parse("0.3.6").release,
+ version.parse("0.3.7").release,
+ version.parse("0.3.8").release,
+ version.parse("0.3.9").release,
+ version.parse("0.4.0").release,
+ version.parse("0.4.1").release,
+ ]
+
+
+def dump(obj, file):
+ """Pickle an object to a file."""
+ Pickler(file, recurse=True).dump(obj)
+
+
+def dumps(obj):
+ """Pickle an object to a string."""
+ file = BytesIO()
+ dump(obj, file)
+ return file.getvalue()
+
+
+if config.DILL_VERSION < version.parse("0.3.6"):
+
+ def log(pickler, msg):
+ dill._dill.log.info(msg)
+
+elif _is_supported_dill_version():
+
+ def log(pickler, msg):
+ dill._dill.logger.trace(pickler, msg)
+
+
+@pklregister(set)
+def _save_set(pickler, obj):
+ log(pickler, f"Se: {obj}")
+ try:
+ # Faster, but fails for unorderable elements
+ args = (sorted(obj),)
+ except Exception: # TypeError, decimal.InvalidOperation, etc.
+ from datasets.fingerprint import Hasher
+
+ args = (sorted(obj, key=Hasher.hash),)
+
+ pickler.save_reduce(set, args, obj=obj)
+ log(pickler, "# Se")
+
+
+def _save_regexPattern(pickler, obj):
+ import regex # type: ignore
+
+ log(pickler, f"Re: {obj}")
+ args = (obj.pattern, obj.flags)
+ pickler.save_reduce(regex.compile, args, obj=obj)
+ log(pickler, "# Re")
+
+
+def _save_tiktokenEncoding(pickler, obj):
+ import tiktoken # type: ignore
+
+ log(pickler, f"Enc: {obj}")
+ args = (obj.name, obj._pat_str, obj._mergeable_ranks, obj._special_tokens)
+ pickler.save_reduce(tiktoken.Encoding, args, obj=obj)
+ log(pickler, "# Enc")
+
+
+def _save_torchTensor(pickler, obj):
+ import torch # type: ignore
+
+ # `torch.from_numpy` is not picklable in `torch>=1.11.0`
+ def create_torchTensor(np_array, dtype=None):
+ tensor = torch.from_numpy(np_array)
+ if dtype:
+ tensor = tensor.type(dtype)
+ return tensor
+
+ log(pickler, f"To: {obj}")
+ if obj.dtype == torch.bfloat16:
+ args = (obj.detach().to(torch.float).cpu().numpy(), torch.bfloat16)
+ else:
+ args = (obj.detach().cpu().numpy(),)
+ pickler.save_reduce(create_torchTensor, args, obj=obj)
+ log(pickler, "# To")
+
+
+def _save_torchGenerator(pickler, obj):
+ import torch # type: ignore
+
+ def create_torchGenerator(state):
+ generator = torch.Generator()
+ generator.set_state(state)
+ return generator
+
+ log(pickler, f"Ge: {obj}")
+ args = (obj.get_state(),)
+ pickler.save_reduce(create_torchGenerator, args, obj=obj)
+ log(pickler, "# Ge")
+
+
+def _save_spacyLanguage(pickler, obj):
+ import spacy # type: ignore
+
+ def create_spacyLanguage(config, bytes):
+ lang_cls = spacy.util.get_lang_class(config["nlp"]["lang"])
+ lang_inst = lang_cls.from_config(config)
+ return lang_inst.from_bytes(bytes)
+
+ log(pickler, f"Sp: {obj}")
+ args = (obj.config, obj.to_bytes())
+ pickler.save_reduce(create_spacyLanguage, args, obj=obj)
+ log(pickler, "# Sp")
+
+
+def _save_transformersPreTrainedTokenizerBase(pickler, obj):
+ log(pickler, f"Tok: {obj}")
+ # Ignore the `cache` attribute and make hashing stable.
+ #
+ # Some tokenizers backed by the `tokenizers` library mutate their internal `_tokenizer` state when called
+ # (e.g. by enabling truncation/padding). This can change the serialized bytes across runs and make dataset
+ # fingerprints unstable, which prevents `.map(load_from_cache_file=True)` from reusing cache files.
+ #
+ # For hashing/fingerprinting, we temporarily disable backend truncation/padding to avoid these runtime settings
+ # affecting the fingerprint, then restore the original settings.
+ state = obj.__dict__.copy()
+ if "cache" in state and isinstance(state["cache"], dict):
+ state["cache"] = {}
+ if "deprecation_warnings" in state and isinstance(state["deprecation_warnings"], dict):
+ state["deprecation_warnings"] = {}
+
+ backend_tokenizer = obj.__dict__.get("_tokenizer")
+ truncation = padding = None
+ if (
+ backend_tokenizer is not None
+ and hasattr(backend_tokenizer, "truncation")
+ and hasattr(backend_tokenizer, "padding")
+ ):
+ truncation = backend_tokenizer.truncation
+ padding = backend_tokenizer.padding
+ try:
+ if truncation is not None and hasattr(backend_tokenizer, "no_truncation"):
+ backend_tokenizer.no_truncation()
+ if padding is not None and hasattr(backend_tokenizer, "no_padding"):
+ backend_tokenizer.no_padding()
+ except Exception:
+ truncation = padding = None
+
+ try:
+ pickler.save_reduce(type(obj), (), state=state, obj=obj)
+ finally:
+ try:
+ if backend_tokenizer is not None:
+ if truncation is not None and hasattr(backend_tokenizer, "enable_truncation"):
+ backend_tokenizer.enable_truncation(**truncation)
+ if padding is not None and hasattr(backend_tokenizer, "enable_padding"):
+ backend_tokenizer.enable_padding(**padding)
+ except Exception:
+ pass
+ log(pickler, "# Tok")
+
+
+if config.DILL_VERSION < version.parse("0.3.6"):
+
+ @pklregister(CodeType)
+ def _save_code(pickler, obj):
+ """
+ From dill._dill.save_code
+ This is a modified version that removes the origin (filename + line no.)
+ of functions created in notebooks or shells for example.
+ """
+ dill._dill.log.info(f"Co: {obj}")
+ # The filename of a function is the .py file where it is defined.
+ # Filenames of functions created in notebooks or shells start with '<'
+ # ex: for ipython, and for shell
+ # Filenames of functions created in ipykernel the filename
+ # look like f"{tempdir}/ipykernel_{id1}/{id2}.py"
+ # Moreover lambda functions have a special name: ''
+ # ex: (lambda x: x).__code__.co_name == "" # True
+ #
+ # For the hashing mechanism we ignore where the function has been defined
+ # More specifically:
+ # - we ignore the filename of special functions (filename starts with '<')
+ # - we always ignore the line number
+ # - we only use the base name of the file instead of the whole path,
+ # to be robust in case a script is moved for example.
+ #
+ # Only those two lines are different from the original implementation:
+ co_filename = (
+ ""
+ if obj.co_filename.startswith("<")
+ or (
+ len(obj.co_filename.split(os.path.sep)) > 1
+ and obj.co_filename.split(os.path.sep)[-2].startswith("ipykernel_")
+ )
+ or obj.co_name == ""
+ else os.path.basename(obj.co_filename)
+ )
+ co_firstlineno = 1
+ # The rest is the same as in the original dill implementation (with also a version check for 3.10)
+ if dill._dill.PY3:
+ if hasattr(obj, "co_posonlyargcount"): # python 3.8 (16 args)
+ args = (
+ obj.co_argcount,
+ obj.co_posonlyargcount,
+ obj.co_kwonlyargcount,
+ obj.co_nlocals,
+ obj.co_stacksize,
+ obj.co_flags,
+ obj.co_code,
+ obj.co_consts,
+ obj.co_names,
+ obj.co_varnames,
+ co_filename,
+ obj.co_name,
+ co_firstlineno,
+ obj.co_linetable if sys.version_info >= (3, 10) else obj.co_lnotab,
+ obj.co_freevars,
+ obj.co_cellvars,
+ )
+ else: # python 3.7 (15 args)
+ args = (
+ obj.co_argcount,
+ obj.co_kwonlyargcount,
+ obj.co_nlocals,
+ obj.co_stacksize,
+ obj.co_flags,
+ obj.co_code,
+ obj.co_consts,
+ obj.co_names,
+ obj.co_varnames,
+ co_filename,
+ obj.co_name,
+ co_firstlineno,
+ obj.co_lnotab,
+ obj.co_freevars,
+ obj.co_cellvars,
+ )
+ else:
+ args = (
+ obj.co_argcount,
+ obj.co_nlocals,
+ obj.co_stacksize,
+ obj.co_flags,
+ obj.co_code,
+ obj.co_consts,
+ obj.co_names,
+ obj.co_varnames,
+ co_filename,
+ obj.co_name,
+ co_firstlineno,
+ obj.co_lnotab,
+ obj.co_freevars,
+ obj.co_cellvars,
+ )
+ pickler.save_reduce(CodeType, args, obj=obj)
+ dill._dill.log.info("# Co")
+ return
+
+elif _is_supported_dill_version():
+ # From: https://github.com/uqfoundation/dill/blob/dill-0.3.6/dill/_dill.py#L1104
+ @pklregister(CodeType)
+ def save_code(pickler, obj):
+ dill._dill.logger.trace(pickler, "Co: %s", obj)
+
+ ############################################################################################################
+ # Modification here for huggingface/datasets
+ # The filename of a function is the .py file where it is defined.
+ # Filenames of functions created in notebooks or shells start with '<'
+ # ex: for ipython, and for shell
+ # Filenames of functions created in ipykernel the filename
+ # look like f"{tempdir}/ipykernel_{id1}/{id2}.py"
+ # Moreover lambda functions have a special name: ''
+ # ex: (lambda x: x).__code__.co_name == "" # True
+ #
+ # For the hashing mechanism we ignore where the function has been defined
+ # More specifically:
+ # - we ignore the filename of special functions (filename starts with '<')
+ # - we always ignore the line number
+ # - we only use the base name of the file instead of the whole path,
+ # to be robust in case a script is moved for example.
+ #
+ # Only those two lines are different from the original implementation:
+ co_filename = (
+ ""
+ if obj.co_filename.startswith("<")
+ or (
+ len(obj.co_filename.split(os.path.sep)) > 1
+ and obj.co_filename.split(os.path.sep)[-2].startswith("ipykernel_")
+ )
+ or obj.co_name == ""
+ else os.path.basename(obj.co_filename)
+ )
+ co_firstlineno = 1
+ # The rest is the same as in the original dill implementation, except for the replacements:
+ # - obj.co_filename => co_filename
+ # - obj.co_firstlineno => co_firstlineno
+ # - obj.co_lnotab => obj.co_linetable for >= 3.10 since co_lnotab was deprecated
+ ############################################################################################################
+
+ if hasattr(obj, "co_endlinetable"): # python 3.11a (20 args)
+ args = (
+ obj.co_linetable, # Modification for huggingface/datasets ############################################
+ obj.co_argcount,
+ obj.co_posonlyargcount,
+ obj.co_kwonlyargcount,
+ obj.co_nlocals,
+ obj.co_stacksize,
+ obj.co_flags,
+ obj.co_code,
+ obj.co_consts,
+ obj.co_names,
+ obj.co_varnames,
+ co_filename, # Modification for huggingface/datasets ############################################
+ obj.co_name,
+ obj.co_qualname,
+ co_firstlineno, # Modification for huggingface/datasets #########################################
+ obj.co_linetable,
+ obj.co_endlinetable,
+ obj.co_columntable,
+ obj.co_exceptiontable,
+ obj.co_freevars,
+ obj.co_cellvars,
+ )
+ elif hasattr(obj, "co_exceptiontable"): # python 3.11 (18 args)
+ args = (
+ obj.co_linetable, # Modification for huggingface/datasets #######################################
+ obj.co_argcount,
+ obj.co_posonlyargcount,
+ obj.co_kwonlyargcount,
+ obj.co_nlocals,
+ obj.co_stacksize,
+ obj.co_flags,
+ obj.co_code,
+ obj.co_consts,
+ obj.co_names,
+ obj.co_varnames,
+ co_filename, # Modification for huggingface/datasets ############################################
+ obj.co_name,
+ obj.co_qualname,
+ co_firstlineno, # Modification for huggingface/datasets #########################################
+ obj.co_linetable,
+ obj.co_exceptiontable,
+ obj.co_freevars,
+ obj.co_cellvars,
+ )
+ elif hasattr(obj, "co_linetable"): # python 3.10 (16 args)
+ args = (
+ obj.co_linetable, # Modification for huggingface/datasets #######################################
+ obj.co_argcount,
+ obj.co_posonlyargcount,
+ obj.co_kwonlyargcount,
+ obj.co_nlocals,
+ obj.co_stacksize,
+ obj.co_flags,
+ obj.co_code,
+ obj.co_consts,
+ obj.co_names,
+ obj.co_varnames,
+ co_filename, # Modification for huggingface/datasets ############################################
+ obj.co_name,
+ co_firstlineno, # Modification for huggingface/datasets #########################################
+ obj.co_linetable,
+ obj.co_freevars,
+ obj.co_cellvars,
+ )
+ elif hasattr(obj, "co_posonlyargcount"): # python 3.8 (16 args)
+ args = (
+ obj.co_argcount,
+ obj.co_posonlyargcount,
+ obj.co_kwonlyargcount,
+ obj.co_nlocals,
+ obj.co_stacksize,
+ obj.co_flags,
+ obj.co_code,
+ obj.co_consts,
+ obj.co_names,
+ obj.co_varnames,
+ co_filename, # Modification for huggingface/datasets ############################################
+ obj.co_name,
+ co_firstlineno, # Modification for huggingface/datasets #########################################
+ obj.co_lnotab,
+ obj.co_freevars,
+ obj.co_cellvars,
+ )
+ else: # python 3.7 (15 args)
+ args = (
+ obj.co_argcount,
+ obj.co_kwonlyargcount,
+ obj.co_nlocals,
+ obj.co_stacksize,
+ obj.co_flags,
+ obj.co_code,
+ obj.co_consts,
+ obj.co_names,
+ obj.co_varnames,
+ co_filename, # Modification for huggingface/datasets ############################################
+ obj.co_name,
+ co_firstlineno, # Modification for huggingface/datasets #########################################
+ obj.co_lnotab,
+ obj.co_freevars,
+ obj.co_cellvars,
+ )
+
+ pickler.save_reduce(dill._dill._create_code, args, obj=obj)
+ dill._dill.logger.trace(pickler, "# Co")
+ return
diff --git a/venv/Lib/site-packages/datasets/utils/_filelock.py b/venv/Lib/site-packages/datasets/utils/_filelock.py
new file mode 100644
index 0000000000000000000000000000000000000000..803ce77f5a4168ac9023e1d4a4548db2bd154126
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/_filelock.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+"""Utilities to handle file locking in `datasets`."""
+
+import os
+
+from filelock import FileLock as FileLock_
+from filelock import UnixFileLock
+from filelock import __version__ as _filelock_version
+from packaging import version
+
+
+class FileLock(FileLock_):
+ """
+ A `filelock.FileLock` initializer that handles long paths.
+ It also uses the current umask for lock files.
+ """
+
+ MAX_FILENAME_LENGTH = 255
+
+ def __init__(self, lock_file, *args, **kwargs):
+ # The "mode" argument is required if we want to use the current umask in filelock >= 3.10
+ # In previous previous it was already using the current umask.
+ if "mode" not in kwargs and version.parse(_filelock_version) >= version.parse("3.10.0"):
+ umask = os.umask(0o666)
+ os.umask(umask)
+ kwargs["mode"] = 0o666 & ~umask
+ lock_file = self.hash_filename_if_too_long(lock_file)
+ super().__init__(lock_file, *args, **kwargs)
+
+ @classmethod
+ def hash_filename_if_too_long(cls, path: str) -> str:
+ path = os.path.abspath(os.path.expanduser(path))
+ filename = os.path.basename(path)
+ max_filename_length = cls.MAX_FILENAME_LENGTH
+ if issubclass(cls, UnixFileLock):
+ max_filename_length = min(max_filename_length, os.statvfs(os.path.dirname(path)).f_namemax)
+ if len(filename) > max_filename_length:
+ dirname = os.path.dirname(path)
+ hashed_filename = str(hash(filename))
+ new_filename = (
+ filename[: max_filename_length - len(hashed_filename) - 8] + "..." + hashed_filename + ".lock"
+ )
+ return os.path.join(dirname, new_filename)
+ else:
+ return path
diff --git a/venv/Lib/site-packages/datasets/utils/deprecation_utils.py b/venv/Lib/site-packages/datasets/utils/deprecation_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..f05ecbeaa3eae5476e99c461dbede9ebfa111eb0
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/deprecation_utils.py
@@ -0,0 +1,105 @@
+import enum
+import inspect
+import warnings
+from functools import wraps
+from typing import Callable, Optional
+
+from .logging import get_logger
+
+
+_emitted_deprecation_warnings = set()
+logger = get_logger(__name__)
+
+
+def deprecated(help_message: Optional[str] = None):
+ """Decorator to mark a class or a function as deprecated.
+
+ Args:
+ help_message (:obj:`str`, optional): An optional message to guide the user on how to
+ switch to non-deprecated usage of the library.
+ """
+
+ def decorator(deprecated_class_or_function: Callable):
+ global _emitted_deprecation_warnings
+
+ if inspect.isclass(deprecated_class_or_function):
+ deprecated_function = deprecated_class_or_function.__init__
+ name = deprecated_class_or_function.__name__
+ else:
+ deprecated_function = deprecated_class_or_function
+ name = deprecated_function.__name__
+ # Support deprecating __init__ class method: class name instead
+ name = name if name != "__init__" else deprecated_function.__qualname__.split(".")[-2]
+
+ warning_msg = (
+ f"{name} is deprecated and will be removed in the next major version of datasets." + f" {help_message}"
+ if help_message
+ else ""
+ )
+
+ @wraps(deprecated_function)
+ def wrapper(*args, **kwargs):
+ func_hash = hash(deprecated_function)
+ if func_hash not in _emitted_deprecation_warnings:
+ warnings.warn(warning_msg, category=FutureWarning, stacklevel=2)
+ _emitted_deprecation_warnings.add(func_hash)
+ return deprecated_function(*args, **kwargs)
+
+ wrapper._decorator_name_ = "deprecated"
+
+ if inspect.isclass(deprecated_class_or_function):
+ deprecated_class_or_function.__init__ = wrapper
+ return deprecated_class_or_function
+ else:
+ return wrapper
+
+ return decorator
+
+
+class OnAccess(enum.EnumMeta):
+ """
+ Enum metaclass that calls a user-specified function whenever a member is accessed.
+ """
+
+ def __getattribute__(cls, name):
+ obj = super().__getattribute__(name)
+ if isinstance(obj, enum.Enum) and obj._on_access:
+ obj._on_access()
+ return obj
+
+ def __getitem__(cls, name):
+ member = super().__getitem__(name)
+ if member._on_access:
+ member._on_access()
+ return member
+
+ def __call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1):
+ obj = super().__call__(value, names, module=module, qualname=qualname, type=type, start=start)
+ if isinstance(obj, enum.Enum) and obj._on_access:
+ obj._on_access()
+ return obj
+
+
+class DeprecatedEnum(enum.Enum, metaclass=OnAccess):
+ """
+ Enum class that calls `deprecate` method whenever a member is accessed.
+ """
+
+ def __new__(cls, value):
+ member = object.__new__(cls)
+ member._value_ = value
+ member._on_access = member.deprecate
+ return member
+
+ @property
+ def help_message(self):
+ return ""
+
+ def deprecate(self):
+ help_message = f" {self.help_message}" if self.help_message else ""
+ warnings.warn(
+ f"'{self.__objclass__.__name__}' is deprecated and will be removed in the next major version of datasets."
+ + help_message,
+ FutureWarning,
+ stacklevel=3,
+ )
diff --git a/venv/Lib/site-packages/datasets/utils/doc_utils.py b/venv/Lib/site-packages/datasets/utils/doc_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..6ef8bcb4e70725ad086cb817e0ec4551d1c0966e
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/doc_utils.py
@@ -0,0 +1,15 @@
+from typing import Callable
+
+
+def is_documented_by(function_with_docstring: Callable):
+ """Decorator to share docstrings across common functions.
+
+ Args:
+ function_with_docstring (`Callable`): Name of the function with the docstring.
+ """
+
+ def wrapper(target_function):
+ target_function.__doc__ = function_with_docstring.__doc__
+ return target_function
+
+ return wrapper
diff --git a/venv/Lib/site-packages/datasets/utils/experimental.py b/venv/Lib/site-packages/datasets/utils/experimental.py
new file mode 100644
index 0000000000000000000000000000000000000000..cc406154e9347f4df83b1f7b08c32a961d469f6a
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/experimental.py
@@ -0,0 +1,43 @@
+"""Contains utilities to flag a feature as "experimental" in datasets."""
+
+import warnings
+from functools import wraps
+from typing import Callable
+
+
+def experimental(fn: Callable) -> Callable:
+ """Decorator to flag a feature as experimental.
+
+ An experimental feature trigger a warning when used as it might be subject to breaking changes in the future.
+
+ Args:
+ fn (`Callable`):
+ The function to flag as experimental.
+
+ Returns:
+ `Callable`: The decorated function.
+
+ Example:
+
+ ```python
+ >>> from datasets.utils import experimental
+
+ >>> @experimental
+ ... def my_function():
+ ... print("Hello world!")
+
+ >>> my_function()
+ UserWarning: 'my_function' is experimental and might be subject to breaking changes in the future.
+ Hello world!
+ ```
+ """
+
+ @wraps(fn)
+ def _inner_fn(*args, **kwargs):
+ warnings.warn(
+ (f"'{fn.__name__}' is experimental and might be subject to breaking changes in the future."),
+ UserWarning,
+ )
+ return fn(*args, **kwargs)
+
+ return _inner_fn
diff --git a/venv/Lib/site-packages/datasets/utils/extract.py b/venv/Lib/site-packages/datasets/utils/extract.py
new file mode 100644
index 0000000000000000000000000000000000000000..c9d96284e5adbd2372aab38717ec6016a9311ffd
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/extract.py
@@ -0,0 +1,444 @@
+import bz2
+import gzip
+import lzma
+import os
+import shutil
+import struct
+import tarfile
+import warnings
+import zipfile
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional, Union
+
+from .. import config
+from ._filelock import FileLock
+from .logging import get_logger
+
+
+if TYPE_CHECKING:
+ import py7zr
+ import rarfile
+
+
+logger = get_logger(__name__)
+
+
+class ExtractManager:
+ def __init__(self, cache_dir: Optional[str] = None):
+ self.extract_dir = (
+ os.path.join(cache_dir, config.EXTRACTED_DATASETS_DIR) if cache_dir else config.EXTRACTED_DATASETS_PATH
+ )
+ self.extractor = Extractor
+
+ def _get_output_path(self, path: str) -> str:
+ from .file_utils import hash_url_to_filename
+
+ # Path where we extract compressed archives
+ # We extract in the cache dir, and get the extracted path name by hashing the original path"
+ abs_path = os.path.abspath(path)
+ return os.path.join(self.extract_dir, hash_url_to_filename(abs_path))
+
+ def _do_extract(self, output_path: str, force_extract: bool) -> bool:
+ return force_extract or (
+ not os.path.isfile(output_path) and not (os.path.isdir(output_path) and os.listdir(output_path))
+ )
+
+ def extract(self, input_path: str, force_extract: bool = False) -> str:
+ extractor_format = self.extractor.infer_extractor_format(input_path)
+ if not extractor_format:
+ return input_path
+ output_path = self._get_output_path(input_path)
+ if self._do_extract(output_path, force_extract):
+ self.extractor.extract(input_path, output_path, extractor_format)
+ return output_path
+
+
+class BaseExtractor(ABC):
+ @classmethod
+ @abstractmethod
+ def is_extractable(cls, path: Union[Path, str], **kwargs) -> bool: ...
+
+ @staticmethod
+ @abstractmethod
+ def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: ...
+
+
+class MagicNumberBaseExtractor(BaseExtractor, ABC):
+ magic_numbers: list[bytes] = []
+
+ @staticmethod
+ def read_magic_number(path: Union[Path, str], magic_number_length: int):
+ with open(path, "rb") as f:
+ return f.read(magic_number_length)
+
+ @classmethod
+ def is_extractable(cls, path: Union[Path, str], magic_number: bytes = b"") -> bool:
+ if not magic_number:
+ magic_number_length = max(len(cls_magic_number) for cls_magic_number in cls.magic_numbers)
+ try:
+ magic_number = cls.read_magic_number(path, magic_number_length)
+ except OSError:
+ return False
+ return any(magic_number.startswith(cls_magic_number) for cls_magic_number in cls.magic_numbers)
+
+
+class TarExtractor(BaseExtractor):
+ @classmethod
+ def is_extractable(cls, path: Union[Path, str], **kwargs) -> bool:
+ return tarfile.is_tarfile(path)
+
+ @staticmethod
+ def safemembers(members: tarfile.TarFile, output_path: Union[Path, str]):
+ """
+ Fix for CVE-2007-4559
+ Desc:
+ Directory traversal vulnerability in the (1) extract and (2) extractall functions in the tarfile
+ module in Python allows user-assisted remote attackers to overwrite arbitrary files via a .. (dot dot)
+ sequence in filenames in a TAR archive, a related issue to CVE-2001-1267.
+ See: https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2007-4559
+ From: https://stackoverflow.com/a/10077309
+ """
+
+ def resolved(path: Union[Path, str]) -> str:
+ return os.path.realpath(os.path.abspath(path))
+
+ def badpath(path: str, base: str) -> bool:
+ # joinpath will ignore base if path is absolute
+ return not resolved(os.path.join(base, path)).startswith(base)
+
+ def badlink(info: tarfile.TarInfo, base: str) -> bool:
+ # Links are interpreted relative to the directory containing the link
+ tip = resolved(os.path.join(base, os.path.dirname(info.name)))
+ return badpath(info.linkname, base=tip)
+
+ base = resolved(output_path)
+
+ for finfo in members:
+ if badpath(finfo.name, base):
+ logger.error(f"Extraction of {finfo.name} is blocked (illegal path)")
+ elif finfo.issym() and badlink(finfo, base):
+ logger.error(f"Extraction of {finfo.name} is blocked: Symlink to {finfo.linkname}")
+ elif finfo.islnk() and badlink(finfo, base):
+ logger.error(f"Extraction of {finfo.name} is blocked: Hard link to {finfo.linkname}")
+ else:
+ yield finfo
+
+ @staticmethod
+ def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None:
+ os.makedirs(output_path, exist_ok=True)
+ tar_file = tarfile.open(input_path)
+ tar_file.extractall(output_path, members=TarExtractor.safemembers(tar_file, output_path))
+ tar_file.close()
+
+
+class GzipExtractor(MagicNumberBaseExtractor):
+ magic_numbers = [b"\x1f\x8b"]
+
+ @staticmethod
+ def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None:
+ with gzip.open(input_path, "rb") as gzip_file:
+ with open(output_path, "wb") as extracted_file:
+ shutil.copyfileobj(gzip_file, extracted_file)
+
+
+class ZipExtractor(MagicNumberBaseExtractor):
+ magic_numbers = [
+ b"PK\x03\x04",
+ b"PK\x05\x06", # empty archive
+ b"PK\x07\x08", # spanned archive
+ ]
+
+ @classmethod
+ def is_extractable(cls, path: Union[Path, str], magic_number: bytes = b"") -> bool:
+ if super().is_extractable(path, magic_number=magic_number):
+ return True
+ try:
+ # Alternative version of zipfile.is_zipfile that has less false positives, but misses executable zip archives.
+ # From: https://github.com/python/cpython/pull/5053
+ from zipfile import (
+ _CD_SIGNATURE,
+ _ECD_DISK_NUMBER,
+ _ECD_DISK_START,
+ _ECD_ENTRIES_TOTAL,
+ _ECD_OFFSET,
+ _ECD_SIZE,
+ _EndRecData,
+ sizeCentralDir,
+ stringCentralDir,
+ structCentralDir,
+ )
+
+ with open(path, "rb") as fp:
+ endrec = _EndRecData(fp)
+ if endrec:
+ if endrec[_ECD_ENTRIES_TOTAL] == 0 and endrec[_ECD_SIZE] == 0 and endrec[_ECD_OFFSET] == 0:
+ return True # Empty zipfiles are still zipfiles
+ elif endrec[_ECD_DISK_NUMBER] == endrec[_ECD_DISK_START]:
+ fp.seek(endrec[_ECD_OFFSET]) # Central directory is on the same disk
+ if fp.tell() == endrec[_ECD_OFFSET] and endrec[_ECD_SIZE] >= sizeCentralDir:
+ data = fp.read(sizeCentralDir) # CD is where we expect it to be
+ if len(data) == sizeCentralDir:
+ centdir = struct.unpack(structCentralDir, data) # CD is the right size
+ if centdir[_CD_SIGNATURE] == stringCentralDir:
+ return True # First central directory entry has correct magic number
+ return False
+ except Exception: # catch all errors in case future python versions change the zipfile internals
+ return False
+
+ @staticmethod
+ def safemembers(members: list[zipfile.ZipInfo], output_path: Union[Path, str]):
+ """
+ Fix for CVE-2007-4559
+ Desc:
+ Directory traversal vulnerability in the (1) extract and (2) extractall functions in the tarfile
+ module in Python allows user-assisted remote attackers to overwrite arbitrary files via a .. (dot dot)
+ sequence in filenames in a TAR archive, a related issue to CVE-2001-1267.
+ See: https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2007-4559
+ From: https://stackoverflow.com/a/10077309
+
+ This additional mitigation is applied for zipfile as well.
+ """
+
+ def resolved(path: Union[Path, str]) -> str:
+ return os.path.realpath(os.path.abspath(path))
+
+ def badpath(path: str, base: str) -> bool:
+ # joinpath will ignore base if path is absolute
+ return not resolved(os.path.join(base, path)).startswith(base)
+
+ base = resolved(output_path)
+
+ for finfo in members:
+ if badpath(finfo.filename, base):
+ logger.error(f"Extraction of {finfo.filename} is blocked (illegal path)")
+ # zipfile doesn't support symlinks
+ # elif finfo.is_symlink and badlink(finfo, base):
+ # logger.error(f"Extraction of {finfo.name} is blocked: Symlink to {finfo.linkname}")
+ else:
+ yield finfo
+
+ @staticmethod
+ def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None:
+ os.makedirs(output_path, exist_ok=True)
+ with zipfile.ZipFile(input_path, "r") as zip_file:
+ zip_file.extractall(output_path, members=ZipExtractor.safemembers(zip_file.filelist, output_path))
+ zip_file.close()
+
+
+class XzExtractor(MagicNumberBaseExtractor):
+ magic_numbers = [b"\xfd\x37\x7a\x58\x5a\x00"]
+
+ @staticmethod
+ def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None:
+ with lzma.open(input_path) as compressed_file:
+ with open(output_path, "wb") as extracted_file:
+ shutil.copyfileobj(compressed_file, extracted_file)
+
+
+class RarExtractor(MagicNumberBaseExtractor):
+ magic_numbers = [b"Rar!\x1a\x07\x00", b"Rar!\x1a\x07\x01\x00"] # RAR_ID # RAR5_ID
+
+ @staticmethod
+ def safemembers(members: list["rarfile.RarInfo"], output_path: Union[Path, str]):
+ """
+ Fix for CVE-2007-4559
+ Desc:
+ Directory traversal vulnerability in the (1) extract and (2) extractall functions in the tarfile
+ module in Python allows user-assisted remote attackers to overwrite arbitrary files via a .. (dot dot)
+ sequence in filenames in a TAR archive, a related issue to CVE-2001-1267.
+ See: https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2007-4559
+ From: https://stackoverflow.com/a/10077309
+
+ This additional mitigation is applied for rarfile as well.
+ """
+
+ def resolved(path: Union[Path, str]) -> str:
+ return os.path.realpath(os.path.abspath(path))
+
+ def badpath(path: str, base: str) -> bool:
+ # joinpath will ignore base if path is absolute
+ return not resolved(os.path.join(base, path)).startswith(base)
+
+ def badlink(info: "rarfile.RarInfo", base: str) -> bool:
+ # Links are interpreted relative to the directory containing the link
+ tip = resolved(os.path.join(base, os.path.dirname(info.filename)))
+ redir_type, redir_flags, link_name = info.file_redir
+ return badpath(link_name, base=tip)
+
+ base = resolved(output_path)
+
+ for finfo in members:
+ if badpath(finfo.filename, base):
+ logger.error(f"Extraction of {finfo.filename} is blocked (illegal path)")
+ elif finfo.is_symlink() and badlink(finfo, base):
+ logger.error(f"Extraction of {finfo.filename} is blocked: Symlink to {finfo.file_redir}")
+ else:
+ yield finfo
+
+ @staticmethod
+ def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None:
+ if not config.RARFILE_AVAILABLE:
+ raise ImportError("Please pip install rarfile")
+ import rarfile
+
+ os.makedirs(output_path, exist_ok=True)
+ rf = rarfile.RarFile(input_path)
+ rf.extractall(output_path, members=RarExtractor.safemembers(rf.infolist(), output_path))
+ rf.close()
+
+
+class ZstdExtractor(MagicNumberBaseExtractor):
+ magic_numbers = [b"\x28\xb5\x2f\xfd"]
+
+ @staticmethod
+ def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None:
+ if not config.ZSTANDARD_AVAILABLE:
+ raise ImportError("Please pip install zstandard")
+ import zstandard as zstd
+
+ dctx = zstd.ZstdDecompressor()
+ with open(input_path, "rb") as ifh, open(output_path, "wb") as ofh:
+ dctx.copy_stream(ifh, ofh)
+
+
+class Bzip2Extractor(MagicNumberBaseExtractor):
+ magic_numbers = [b"\x42\x5a\x68"]
+
+ @staticmethod
+ def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None:
+ with bz2.open(input_path, "rb") as compressed_file:
+ with open(output_path, "wb") as extracted_file:
+ shutil.copyfileobj(compressed_file, extracted_file)
+
+
+class SevenZipExtractor(MagicNumberBaseExtractor):
+ magic_numbers = [b"\x37\x7a\xbc\xaf\x27\x1c"]
+
+ @staticmethod
+ def safemembers(members: list["py7zr.FileInfo"], output_path: Union[Path, str]):
+ """
+ Fix for CVE-2007-4559
+ Desc:
+ Directory traversal vulnerability in the (1) extract and (2) extractall functions in the tarfile
+ module in Python allows user-assisted remote attackers to overwrite arbitrary files via a .. (dot dot)
+ sequence in filenames in a TAR archive, a related issue to CVE-2001-1267.
+ See: https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2007-4559
+ From: https://stackoverflow.com/a/10077309
+
+ This additional mitigation is applied for py7zr as well.
+ """
+
+ def resolved(path: Union[Path, str]) -> str:
+ return os.path.realpath(os.path.abspath(path))
+
+ def badpath(path: str, base: str) -> bool:
+ # joinpath will ignore base if path is absolute
+ return not resolved(os.path.join(base, path)).startswith(base)
+
+ def badlink(info: "py7zr.FileInfo", base: str) -> bool:
+ # Links are interpreted relative to the directory containing the link
+ tip = resolved(os.path.join(base, os.path.dirname(info.filename)))
+ return badpath(os.path.basename(info.filename), base=tip)
+
+ base = resolved(output_path)
+
+ for finfo in members:
+ if badpath(finfo.filename, base):
+ logger.error(f"Extraction of {finfo.filename} is blocked (illegal path)")
+ # py7zr already checks symlinks validity
+ # elif finfo.is_symlink and badlink(finfo, base):
+ # logger.error(f"Extraction of {finfo.name} is blocked: Symlink to {finfo.linkname}")
+ else:
+ yield finfo
+
+ @staticmethod
+ def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None:
+ if not config.PY7ZR_AVAILABLE:
+ raise ImportError("Please pip install py7zr")
+ import py7zr
+
+ os.makedirs(output_path, exist_ok=True)
+ with py7zr.SevenZipFile(input_path, "r") as archive:
+ targets = [finfo.filename for finfo in SevenZipExtractor.safemembers(archive.list(), output_path)]
+ archive.extract(output_path, targets=targets)
+
+
+class Lz4Extractor(MagicNumberBaseExtractor):
+ magic_numbers = [b"\x04\x22\x4d\x18"]
+
+ @staticmethod
+ def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None:
+ if not config.LZ4_AVAILABLE:
+ raise ImportError("Please pip install lz4")
+ import lz4.frame
+
+ with lz4.frame.open(input_path, "rb") as compressed_file:
+ with open(output_path, "wb") as extracted_file:
+ shutil.copyfileobj(compressed_file, extracted_file)
+
+
+class Extractor:
+ # Put zip file to the last, b/c it is possible wrongly detected as zip (I guess it means: as tar or gzip)
+ extractors: dict[str, type[BaseExtractor]] = {
+ "tar": TarExtractor,
+ "gzip": GzipExtractor,
+ "zip": ZipExtractor,
+ "xz": XzExtractor,
+ "rar": RarExtractor,
+ "zstd": ZstdExtractor,
+ "bz2": Bzip2Extractor,
+ "7z": SevenZipExtractor, #
+ "lz4": Lz4Extractor, #
+ }
+
+ @classmethod
+ def _get_magic_number_max_length(cls):
+ return max(
+ len(extractor_magic_number)
+ for extractor in cls.extractors.values()
+ if issubclass(extractor, MagicNumberBaseExtractor)
+ for extractor_magic_number in extractor.magic_numbers
+ )
+
+ @staticmethod
+ def _read_magic_number(path: Union[Path, str], magic_number_length: int):
+ try:
+ return MagicNumberBaseExtractor.read_magic_number(path, magic_number_length=magic_number_length)
+ except OSError:
+ return b""
+
+ @classmethod
+ def is_extractable(cls, path: Union[Path, str], return_extractor: bool = False) -> bool:
+ warnings.warn(
+ "Method 'is_extractable' was deprecated in version 2.4.0 and will be removed in 3.0.0. "
+ "Use 'infer_extractor_format' instead.",
+ category=FutureWarning,
+ )
+ extractor_format = cls.infer_extractor_format(path)
+ if extractor_format:
+ return True if not return_extractor else (True, cls.extractors[extractor_format])
+ return False if not return_extractor else (False, None)
+
+ @classmethod
+ def infer_extractor_format(cls, path: Union[Path, str]) -> Optional[str]: #
+ magic_number_max_length = cls._get_magic_number_max_length()
+ magic_number = cls._read_magic_number(path, magic_number_max_length)
+ for extractor_format, extractor in cls.extractors.items():
+ if extractor.is_extractable(path, magic_number=magic_number):
+ return extractor_format
+
+ @classmethod
+ def extract(
+ cls,
+ input_path: Union[Path, str],
+ output_path: Union[Path, str],
+ extractor_format: str,
+ ) -> None:
+ os.makedirs(os.path.dirname(output_path), exist_ok=True)
+ # Prevent parallel extractions
+ lock_path = str(Path(output_path).with_suffix(".lock"))
+ with FileLock(lock_path):
+ shutil.rmtree(output_path, ignore_errors=True)
+ extractor = cls.extractors[extractor_format]
+ return extractor.extract(input_path, output_path)
diff --git a/venv/Lib/site-packages/datasets/utils/file_utils.py b/venv/Lib/site-packages/datasets/utils/file_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..9f74b2a52195e59a4584f4e6cf47d554c502c04a
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/file_utils.py
@@ -0,0 +1,1427 @@
+"""
+Utilities for working with the local dataset cache.
+This file is adapted from the AllenNLP library at https://github.com/allenai/allennlp
+Copyright by the AllenNLP authors.
+"""
+
+import asyncio
+import glob
+import io
+import json
+import multiprocessing
+import os
+import posixpath
+import re
+import shutil
+import tarfile
+import time
+import xml.dom.minidom
+import zipfile
+from collections.abc import Generator
+from io import BytesIO
+from itertools import chain
+from pathlib import Path, PurePosixPath
+from typing import Any, Optional, TypeVar, Union
+from unittest.mock import patch
+from urllib.parse import urlparse
+from xml.etree import ElementTree as ET
+
+import fsspec
+import httpx
+import huggingface_hub
+import huggingface_hub.errors
+import requests
+from fsspec.core import strip_protocol, url_to_fs
+from fsspec.utils import can_be_local
+from huggingface_hub.utils import get_session, insecure_hashlib
+from packaging import version
+
+from .. import __version__, config
+from ..download.download_config import DownloadConfig
+from ..filesystems import COMPRESSION_FILESYSTEMS
+from . import _tqdm, logging
+from ._filelock import FileLock
+from .extract import ExtractManager
+from .track import TrackedIterableFromGenerator
+
+
+try:
+ from aiohttp.client_exceptions import ClientError as _AiohttpClientError
+except ImportError:
+ # aiohttp is not available; synthesize an exception type
+ # that will never be raised by any actual code for use in the `except`
+ # clause only.
+ class _AiohttpClientError(Exception):
+ pass
+
+
+logger = logging.get_logger(__name__) # pylint: disable=invalid-name
+
+INCOMPLETE_SUFFIX = ".incomplete"
+
+T = TypeVar("T", str, Path)
+
+CONNECTION_ERRORS_TO_RETRY = (
+ _AiohttpClientError,
+ asyncio.TimeoutError,
+ requests.exceptions.ConnectionError,
+ requests.exceptions.Timeout,
+ httpx.RequestError,
+)
+SERVER_UNAVAILABLE_CODE = 504
+RATE_LIMIT_CODE = 429
+
+
+def is_remote_url(url_or_filename: str) -> bool:
+ return urlparse(url_or_filename).scheme != "" and not os.path.ismount(urlparse(url_or_filename).scheme + ":/")
+
+
+def is_local_path(url_or_filename: str) -> bool:
+ # On unix the scheme of a local path is empty (for both absolute and relative),
+ # while on windows the scheme is the drive name (ex: "c") for absolute paths.
+ # for details on the windows behavior, see https://bugs.python.org/issue42215
+ return urlparse(url_or_filename).scheme == "" or os.path.ismount(urlparse(url_or_filename).scheme + ":/")
+
+
+def is_relative_path(url_or_filename: str) -> bool:
+ return urlparse(url_or_filename).scheme == "" and not os.path.isabs(url_or_filename)
+
+
+def relative_to_absolute_path(path: T) -> T:
+ """Convert relative path to absolute path."""
+ abs_path_str = os.path.abspath(os.path.expanduser(os.path.expandvars(str(path))))
+ return Path(abs_path_str) if isinstance(path, Path) else abs_path_str
+
+
+def url_or_path_join(base_name: str, *pathnames: str) -> str:
+ if is_remote_url(base_name):
+ return posixpath.join(base_name, *(str(pathname).replace(os.sep, "/").lstrip("/") for pathname in pathnames))
+ else:
+ return Path(base_name, *pathnames).as_posix()
+
+
+def url_or_path_parent(url_or_path: str) -> str:
+ if is_remote_url(url_or_path):
+ return url_or_path[: url_or_path.rindex("/")]
+ else:
+ return os.path.dirname(url_or_path)
+
+
+def hash_url_to_filename(url, etag=None):
+ """
+ Convert `url` into a hashed filename in a repeatable way.
+ If `etag` is specified, append its hash to the url's, delimited
+ by a period.
+ If the url ends with .h5 (Keras HDF5 weights) adds '.h5' to the name
+ so that TF 2.0 can identify it as a HDF5 file
+ (see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1380)
+ """
+ url_bytes = url.encode("utf-8")
+ url_hash = insecure_hashlib.sha256(url_bytes)
+ filename = url_hash.hexdigest()
+
+ if etag:
+ etag_bytes = etag.encode("utf-8")
+ etag_hash = insecure_hashlib.sha256(etag_bytes)
+ filename += "." + etag_hash.hexdigest()
+
+ if url.endswith(".py"):
+ filename += ".py"
+
+ return filename
+
+
+def cached_path(
+ url_or_filename,
+ download_config=None,
+ **download_kwargs,
+) -> str:
+ """
+ Given something that might be a URL (or might be a local path),
+ determine which. If it's a URL, download the file and cache it, and
+ return the path to the cached file. If it's already a local path,
+ make sure the file exists and then return the path.
+
+ Return:
+ Local path (string)
+
+ Raises:
+ FileNotFoundError: in case of non-recoverable file
+ (non-existent or no cache on disk)
+ ConnectionError: in case of unreachable url
+ and no cache on disk
+ ValueError: if it couldn't parse the url or filename correctly
+ httpx.NetworkError or requests.exceptions.ConnectionError: in case of internet connection issue
+ """
+ if download_config is None:
+ download_config = DownloadConfig(**download_kwargs)
+
+ cache_dir = download_config.cache_dir or config.DOWNLOADED_DATASETS_PATH
+ if isinstance(cache_dir, Path):
+ cache_dir = str(cache_dir)
+ if isinstance(url_or_filename, Path):
+ url_or_filename = str(url_or_filename)
+
+ # Convert fsspec URL in the format "file://local/path" to "local/path"
+ if can_be_local(url_or_filename):
+ url_or_filename = strip_protocol(url_or_filename)
+
+ if is_remote_url(url_or_filename):
+ # URL, so get it from the cache (downloading if necessary)
+ url_or_filename, storage_options = _prepare_path_and_storage_options(
+ url_or_filename, download_config=download_config
+ )
+ # Download files from Hugging Face.
+ # Note: no need to check for https://huggingface.co file URLs since _prepare_path_and_storage_options
+ # prepares Hugging Face HTTP URLs as hf:// paths already
+ if url_or_filename.startswith("hf://") and not url_or_filename.startswith("hf://buckets/"):
+ resolved_path = huggingface_hub.HfFileSystem(
+ endpoint=config.HF_ENDPOINT, token=download_config.token
+ ).resolve_path(url_or_filename)
+ try:
+ output_path = huggingface_hub.HfApi(
+ endpoint=config.HF_ENDPOINT,
+ token=download_config.token,
+ library_name="datasets",
+ library_version=__version__,
+ user_agent=get_datasets_user_agent(download_config.user_agent),
+ ).hf_hub_download(
+ repo_id=resolved_path.repo_id,
+ repo_type=resolved_path.repo_type,
+ revision=resolved_path.revision,
+ filename=resolved_path.path_in_repo,
+ force_download=download_config.force_download,
+ proxies=download_config.proxies,
+ )
+ except (
+ huggingface_hub.utils.RepositoryNotFoundError,
+ huggingface_hub.utils.EntryNotFoundError,
+ huggingface_hub.utils.RevisionNotFoundError,
+ huggingface_hub.utils.GatedRepoError,
+ ) as e:
+ raise FileNotFoundError(str(e)) from e
+ # Download external files
+ else:
+ output_path = get_from_cache(
+ url_or_filename,
+ cache_dir=cache_dir,
+ force_download=download_config.force_download,
+ user_agent=download_config.user_agent,
+ use_etag=download_config.use_etag,
+ token=download_config.token,
+ storage_options=storage_options,
+ download_desc=download_config.download_desc,
+ disable_tqdm=download_config.disable_tqdm,
+ )
+ elif os.path.exists(url_or_filename):
+ # File, and it exists.
+ output_path = url_or_filename
+ elif is_local_path(url_or_filename):
+ # File, but it doesn't exist.
+ raise FileNotFoundError(f"Local file {url_or_filename} doesn't exist")
+ else:
+ # Something unknown
+ raise ValueError(f"unable to parse {url_or_filename} as a URL or as a local path")
+
+ if output_path is None:
+ return output_path
+
+ if download_config.extract_compressed_file:
+ if download_config.extract_on_the_fly:
+ # Add a compression prefix to the compressed file so that it can be extracted
+ # as it's being read using xopen.
+ protocol = _get_extraction_protocol(output_path, download_config=download_config)
+ extension = _get_path_extension(url_or_filename.split("::")[0])
+ if (
+ protocol
+ and extension not in ["tgz", "tar"]
+ and not url_or_filename.split("::")[0].endswith((".tar.gz", ".tar.bz2", ".tar.xz"))
+ ):
+ output_path = relative_to_absolute_path(output_path)
+ if protocol in SINGLE_FILE_COMPRESSION_PROTOCOLS:
+ # there is one single file which is the uncompressed file
+ inner_file = os.path.basename(output_path)
+ inner_file = inner_file[: inner_file.rindex(".")] if "." in inner_file else inner_file
+ output_path = f"{protocol}://{inner_file}::{output_path}"
+ else:
+ output_path = f"{protocol}://::{output_path}"
+ return output_path
+
+ # Eager extraction
+ output_path = ExtractManager(cache_dir=download_config.cache_dir).extract(
+ output_path, force_extract=download_config.force_extract
+ )
+ return relative_to_absolute_path(output_path)
+
+
+def get_datasets_user_agent(user_agent: Optional[Union[str, dict]] = None) -> str:
+ ua = f"datasets/{__version__}"
+ ua += f"; python/{config.PY_VERSION}"
+ ua += f"; hf_hub/{huggingface_hub.__version__}"
+ ua += f"; pyarrow/{config.PYARROW_VERSION}"
+ if config.TORCH_AVAILABLE:
+ ua += f"; torch/{config.TORCH_VERSION}"
+ if config.TF_AVAILABLE:
+ ua += f"; tensorflow/{config.TF_VERSION}"
+ if config.JAX_AVAILABLE:
+ ua += f"; jax/{config.JAX_VERSION}"
+ if isinstance(user_agent, dict):
+ ua += f"; {'; '.join(f'{k}/{v}' for k, v in user_agent.items())}"
+ elif isinstance(user_agent, str):
+ ua += "; " + user_agent
+ return ua
+
+
+def get_authentication_headers_for_url(url: str, token: Optional[Union[str, bool]] = None) -> dict:
+ """Handle the HF authentication"""
+ if url.startswith(config.HF_ENDPOINT):
+ return huggingface_hub.utils.build_hf_headers(
+ token=token, library_name="datasets", library_version=__version__
+ )
+ else:
+ return {}
+
+
+def _raise_if_offline_mode_is_enabled(msg: Optional[str] = None):
+ """Raise an OfflineModeIsEnabled error (subclass of ConnectionError) if HF_HUB_OFFLINE is True."""
+ if config.HF_HUB_OFFLINE:
+ raise huggingface_hub.errors.OfflineModeIsEnabled(
+ "Offline mode is enabled." if msg is None else "Offline mode is enabled. " + str(msg)
+ )
+
+
+def fsspec_head(url, storage_options=None):
+ _raise_if_offline_mode_is_enabled(f"Tried to reach {url}")
+ fs, path = url_to_fs(url, **(storage_options or {}))
+ return fs.info(path)
+
+
+def stack_multiprocessing_download_progress_bars():
+ # Stack downloads progress bars automatically using HF_DATASETS_STACK_MULTIPROCESSING_DOWNLOAD_PROGRESS_BARS=1
+ # We use environment variables since the download may happen in a subprocess
+ return patch.dict(os.environ, {"HF_DATASETS_STACK_MULTIPROCESSING_DOWNLOAD_PROGRESS_BARS": "1"})
+
+
+class TqdmCallback(fsspec.callbacks.TqdmCallback):
+ def __init__(self, tqdm_kwargs=None, *args, **kwargs):
+ if config.FSSPEC_VERSION < version.parse("2024.2.0"):
+ super().__init__(tqdm_kwargs, *args, **kwargs)
+ self._tqdm = _tqdm # replace tqdm module by datasets.utils.tqdm module
+ else:
+ kwargs["tqdm_cls"] = _tqdm.tqdm
+ super().__init__(tqdm_kwargs, *args, **kwargs)
+
+
+def fsspec_get(url, temp_file, storage_options=None, desc=None, disable_tqdm=False):
+ _raise_if_offline_mode_is_enabled(f"Tried to reach {url}")
+ fs, path = url_to_fs(url, **(storage_options or {}))
+ callback = TqdmCallback(
+ tqdm_kwargs={
+ "desc": desc or "Downloading",
+ "unit": "B",
+ "unit_scale": True,
+ "position": multiprocessing.current_process()._identity[-1] # contains the ranks of subprocesses
+ if os.environ.get("HF_DATASETS_STACK_MULTIPROCESSING_DOWNLOAD_PROGRESS_BARS") == "1"
+ and multiprocessing.current_process()._identity
+ else None,
+ "disable": disable_tqdm,
+ }
+ )
+ fs.get_file(path, temp_file.name, callback=callback)
+
+
+def get_from_cache(
+ url,
+ cache_dir=None,
+ force_download=False,
+ user_agent=None,
+ use_etag=True,
+ token=None,
+ storage_options=None,
+ download_desc=None,
+ disable_tqdm=False,
+) -> str:
+ """
+ Given a URL, look for the corresponding file in the local cache.
+ If it's not there, download it. Then return the path to the cached file.
+
+ Return:
+ Local path (string)
+
+ Raises:
+ FileNotFoundError: in case of non-recoverable file
+ (non-existent or no cache on disk)
+ ConnectionError: in case of unreachable url
+ and no cache on disk
+ """
+ if storage_options is None:
+ storage_options = {}
+ if cache_dir is None:
+ cache_dir = config.HF_DATASETS_CACHE
+ if isinstance(cache_dir, Path):
+ cache_dir = str(cache_dir)
+
+ os.makedirs(cache_dir, exist_ok=True)
+
+ response = None
+ etag = None
+
+ # Try a first time to file the file on the local file system without eTag (None)
+ # if we don't ask for 'force_download' then we spare a request
+ filename = hash_url_to_filename(url, etag=None)
+ cache_path = os.path.join(cache_dir, filename)
+
+ if os.path.exists(cache_path) and not force_download and not use_etag:
+ return cache_path
+
+ # Prepare headers for authentication
+ headers = get_authentication_headers_for_url(url, token=token)
+ if user_agent is not None:
+ headers["user-agent"] = user_agent
+
+ response = fsspec_head(url, storage_options=storage_options)
+ etag = (response.get("ETag", None) or response.get("etag", None)) if use_etag else None
+
+ # Try a second time
+ filename = hash_url_to_filename(url, etag)
+ cache_path = os.path.join(cache_dir, filename)
+
+ if os.path.exists(cache_path) and not force_download:
+ return cache_path
+
+ # Prevent parallel downloads of the same file with a lock.
+ lock_path = cache_path + ".lock"
+ with FileLock(lock_path):
+ # Retry in case previously locked processes just enter after the precedent process releases the lock
+ if os.path.exists(cache_path) and not force_download:
+ return cache_path
+
+ incomplete_path = cache_path + ".incomplete"
+
+ # Download to temporary file, then copy to cache path once finished.
+ # Otherwise, you get corrupt cache entries if the download gets interrupted.
+ with open(incomplete_path, "w+b") as temp_file:
+ logger.info(f"{url} not found in cache or force_download set to True, downloading to {temp_file.name}")
+ # GET file object
+ fsspec_get(url, temp_file, storage_options=storage_options, desc=download_desc, disable_tqdm=disable_tqdm)
+
+ logger.info(f"storing {url} in cache at {cache_path}")
+ shutil.move(temp_file.name, cache_path)
+
+ logger.info(f"creating metadata file for {cache_path}")
+ meta = {"url": url, "etag": etag}
+ meta_path = cache_path + ".json"
+ with open(meta_path, "w", encoding="utf-8") as meta_file:
+ json.dump(meta, meta_file)
+
+ return cache_path
+
+
+def add_start_docstrings(*docstr):
+ def docstring_decorator(fn):
+ fn.__doc__ = "".join(docstr) + "\n\n" + (fn.__doc__ if fn.__doc__ is not None else "")
+ return fn
+
+ return docstring_decorator
+
+
+def add_end_docstrings(*docstr):
+ def docstring_decorator(fn):
+ fn.__doc__ = (fn.__doc__ if fn.__doc__ is not None else "") + "\n\n" + "".join(docstr)
+ return fn
+
+ return docstring_decorator
+
+
+def estimate_dataset_size(paths):
+ return sum(path.stat().st_size for path in paths)
+
+
+def readline(f: io.RawIOBase):
+ # From: https://github.com/python/cpython/blob/d27e2f4d118e7a9909b6a3e5da06c5ff95806a85/Lib/_pyio.py#L525
+ res = bytearray()
+ while True:
+ b = f.read(1)
+ if not b:
+ break
+ res += b
+ if res.endswith(b"\n"):
+ break
+ return bytes(res)
+
+
+#######################
+# Streaming utilities #
+#######################
+
+BASE_KNOWN_EXTENSIONS = [
+ "txt",
+ "csv",
+ "json",
+ "jsonl",
+ "tsv",
+ "conll",
+ "conllu",
+ "orig",
+ "parquet",
+ "pkl",
+ "pickle",
+ "rel",
+ "xml",
+ "arrow",
+]
+COMPRESSION_EXTENSION_TO_PROTOCOL = {
+ # single file compression
+ **{
+ extension.lstrip("."): fs_class.protocol
+ for fs_class in COMPRESSION_FILESYSTEMS
+ for extension in fs_class.extensions
+ },
+ # archive compression
+ "zip": "zip",
+ "eval": "zip",
+}
+SINGLE_FILE_COMPRESSION_EXTENSION_TO_PROTOCOL = {
+ extension.lstrip("."): fs_class.protocol
+ for fs_class in COMPRESSION_FILESYSTEMS
+ for extension in fs_class.extensions
+}
+SINGLE_FILE_COMPRESSION_PROTOCOLS = {fs_class.protocol for fs_class in COMPRESSION_FILESYSTEMS}
+SINGLE_SLASH_AFTER_PROTOCOL_PATTERN = re.compile(r"(? str:
+ # Get extension: https://foo.bar/train.json.gz -> gz
+ extension = path.split(".")[-1]
+ # Remove query params ("dl=1", "raw=true"): gz?dl=1 -> gz
+ # Remove shards infos (".txt_1", ".txt-00000-of-00100"): txt_1 -> txt
+ for symb in "?-_":
+ extension = extension.split(symb)[0]
+ return extension
+
+
+def _get_extraction_protocol_with_magic_number(f) -> Optional[str]:
+ """read the magic number from a file-like object and return the compression protocol"""
+ # Check if the file object is seekable even before reading the magic number (to avoid https://bugs.python.org/issue26440)
+ try:
+ f.seek(0)
+ except (AttributeError, io.UnsupportedOperation):
+ return None
+ magic_number = f.read(MAGIC_NUMBER_MAX_LENGTH)
+ f.seek(0)
+ for i in range(MAGIC_NUMBER_MAX_LENGTH):
+ compression = MAGIC_NUMBER_TO_COMPRESSION_PROTOCOL.get(magic_number[: MAGIC_NUMBER_MAX_LENGTH - i])
+ if compression is not None:
+ return compression
+ compression = MAGIC_NUMBER_TO_UNSUPPORTED_COMPRESSION_PROTOCOL.get(magic_number[: MAGIC_NUMBER_MAX_LENGTH - i])
+ if compression is not None:
+ raise NotImplementedError(f"Compression protocol '{compression}' not implemented.")
+
+
+def _get_extraction_protocol(urlpath: str, download_config: Optional[DownloadConfig] = None) -> Optional[str]:
+ # get inner file: zip://train-00000.json.gz::https://foo.bar/data.zip -> zip://train-00000.json.gz
+ urlpath = str(urlpath)
+ path = urlpath.split("::")[0]
+ extension = _get_path_extension(path)
+ if (
+ extension in BASE_KNOWN_EXTENSIONS
+ or extension in ["tgz", "tar"]
+ or path.endswith((".tar.gz", ".tar.bz2", ".tar.xz"))
+ ):
+ return None
+ elif extension in COMPRESSION_EXTENSION_TO_PROTOCOL:
+ return COMPRESSION_EXTENSION_TO_PROTOCOL[extension]
+ urlpath, storage_options = _prepare_path_and_storage_options(urlpath, download_config=download_config)
+ try:
+ with fsspec.open(urlpath, **(storage_options or {})) as f:
+ return _get_extraction_protocol_with_magic_number(f)
+ except FileNotFoundError:
+ if urlpath.startswith(config.HF_ENDPOINT):
+ raise FileNotFoundError(
+ urlpath + "\nIf the repo is private or gated, make sure to log in with `huggingface-cli login`."
+ ) from None
+ else:
+ raise
+
+
+def xjoin(a, *p):
+ """
+ This function extends os.path.join to support the "::" hop separator. It supports both paths and urls.
+
+ A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::".
+ This is used to access files inside a zip file over http for example.
+
+ Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt.
+ Then you can just chain the url this way:
+
+ zip://folder1/file.txt::https://host.com/archive.zip
+
+ The xjoin function allows you to apply the join on the first path of the chain.
+
+ Example::
+
+ >>> xjoin("zip://folder1::https://host.com/archive.zip", "file.txt")
+ zip://folder1/file.txt::https://host.com/archive.zip
+ """
+ a, *b = str(a).split("::")
+ if is_local_path(a):
+ return os.path.join(a, *p)
+ else:
+ a = posixpath.join(a, *p)
+ return "::".join([a] + b)
+
+
+def xdirname(a):
+ """
+ This function extends os.path.dirname to support the "::" hop separator. It supports both paths and urls.
+
+ A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::".
+ This is used to access files inside a zip file over http for example.
+
+ Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt.
+ Then you can just chain the url this way:
+
+ zip://folder1/file.txt::https://host.com/archive.zip
+
+ The xdirname function allows you to apply the dirname on the first path of the chain.
+
+ Example::
+
+ >>> xdirname("zip://folder1/file.txt::https://host.com/archive.zip")
+ zip://folder1::https://host.com/archive.zip
+ """
+ a, *b = str(a).split("::")
+ if is_local_path(a):
+ a = os.path.dirname(Path(a).as_posix())
+ else:
+ a = posixpath.dirname(a)
+ # if we end up at the root of the protocol, we get for example a = 'http:'
+ # so we have to fix it by adding the '//' that was removed:
+ if a.endswith(":"):
+ a += "//"
+ return "::".join([a] + b)
+
+
+def xexists(urlpath: str, download_config: Optional[DownloadConfig] = None):
+ """Extend `os.path.exists` function to support both local and remote files.
+
+ Args:
+ urlpath (`str`): URL path.
+ download_config : mainly use token or storage_options to support different platforms and auth types.
+
+ Returns:
+ `bool`
+ """
+
+ main_hop, *rest_hops = _as_str(urlpath).split("::")
+ if is_local_path(main_hop):
+ return os.path.exists(main_hop)
+ else:
+ urlpath, storage_options = _prepare_path_and_storage_options(urlpath, download_config=download_config)
+ main_hop, *rest_hops = urlpath.split("::")
+ fs, *_ = url_to_fs(urlpath, **storage_options)
+ return fs.exists(main_hop)
+
+
+def xbasename(a):
+ """
+ This function extends os.path.basename to support the "::" hop separator. It supports both paths and urls.
+
+ A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::".
+ This is used to access files inside a zip file over http for example.
+
+ Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt.
+ Then you can just chain the url this way:
+
+ zip://folder1/file.txt::https://host.com/archive.zip
+
+ The xbasename function allows you to apply the basename on the first path of the chain.
+
+ Example::
+
+ >>> xbasename("zip://folder1/file.txt::https://host.com/archive.zip")
+ file.txt
+ """
+ a, *b = str(a).split("::")
+ if is_local_path(a):
+ return os.path.basename(Path(a).as_posix())
+ else:
+ return posixpath.basename(a)
+
+
+def xsplit(a):
+ """
+ This function extends os.path.split to support the "::" hop separator. It supports both paths and urls.
+
+ A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::".
+ This is used to access files inside a zip file over http for example.
+
+ Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt.
+ Then you can just chain the url this way:
+
+ zip://folder1/file.txt::https://host.com/archive.zip
+
+ The xsplit function allows you to apply the xsplit on the first path of the chain.
+
+ Example::
+
+ >>> xsplit("zip://folder1/file.txt::https://host.com/archive.zip")
+ ('zip://folder1::https://host.com/archive.zip', 'file.txt')
+ """
+ a, *b = str(a).split("::")
+ if is_local_path(a):
+ return os.path.split(Path(a).as_posix())
+ else:
+ a, tail = posixpath.split(a)
+ return "::".join([a + "//" if a.endswith(":") else a] + b), tail
+
+
+def xsplitext(a):
+ """
+ This function extends os.path.splitext to support the "::" hop separator. It supports both paths and urls.
+
+ A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::".
+ This is used to access files inside a zip file over http for example.
+
+ Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt.
+ Then you can just chain the url this way:
+
+ zip://folder1/file.txt::https://host.com/archive.zip
+
+ The xsplitext function allows you to apply the splitext on the first path of the chain.
+
+ Example::
+
+ >>> xsplitext("zip://folder1/file.txt::https://host.com/archive.zip")
+ ('zip://folder1/file::https://host.com/archive.zip', '.txt')
+ """
+ a, *b = str(a).split("::")
+ if is_local_path(a):
+ return os.path.splitext(Path(a).as_posix())
+ else:
+ a, ext = posixpath.splitext(a)
+ return "::".join([a] + b), ext
+
+
+def xisfile(path, download_config: Optional[DownloadConfig] = None) -> bool:
+ """Extend `os.path.isfile` function to support remote files.
+
+ Args:
+ path (`str`): URL path.
+ download_config : mainly use token or storage_options to support different platforms and auth types.
+
+ Returns:
+ `bool`
+ """
+ main_hop, *rest_hops = str(path).split("::")
+ if is_local_path(main_hop):
+ return os.path.isfile(path)
+ else:
+ path, storage_options = _prepare_path_and_storage_options(path, download_config=download_config)
+ main_hop, *rest_hops = path.split("::")
+ fs, *_ = url_to_fs(path, **storage_options)
+ return fs.isfile(main_hop)
+
+
+def xgetsize(path, download_config: Optional[DownloadConfig] = None) -> int:
+ """Extend `os.path.getsize` function to support remote files.
+
+ Args:
+ path (`str`): URL path.
+ download_config : mainly use token or storage_options to support different platforms and auth types.
+
+ Returns:
+ `int`: optional
+ """
+ main_hop, *rest_hops = str(path).split("::")
+ if is_local_path(main_hop):
+ return os.path.getsize(path)
+ else:
+ path, storage_options = _prepare_path_and_storage_options(path, download_config=download_config)
+ main_hop, *rest_hops = path.split("::")
+ fs, *_ = fs, *_ = url_to_fs(path, **storage_options)
+ try:
+ size = fs.size(main_hop)
+ except huggingface_hub.utils.EntryNotFoundError:
+ raise FileNotFoundError(f"No such file: {path}")
+ if size is None:
+ # use xopen instead of fs.open to make data fetching more robust
+ with xopen(path, download_config=download_config) as f:
+ size = len(f.read())
+ return size
+
+
+def xisdir(path, download_config: Optional[DownloadConfig] = None) -> bool:
+ """Extend `os.path.isdir` function to support remote files.
+
+ Args:
+ path (`str`): URL path.
+ download_config : mainly use token or storage_options to support different platforms and auth types.
+
+ Returns:
+ `bool`
+ """
+ main_hop, *rest_hops = str(path).split("::")
+ if is_local_path(main_hop):
+ return os.path.isdir(path)
+ else:
+ path, storage_options = _prepare_path_and_storage_options(path, download_config=download_config)
+ main_hop, *rest_hops = path.split("::")
+ fs, *_ = fs, *_ = url_to_fs(path, **storage_options)
+ inner_path = main_hop.split("://")[-1]
+ if not inner_path.strip("/"):
+ return True
+ return fs.isdir(inner_path)
+
+
+def xrelpath(path, start=None):
+ """Extend `os.path.relpath` function to support remote files.
+
+ Args:
+ path (`str`): URL path.
+ start (`str`): Start URL directory path.
+
+ Returns:
+ `str`
+ """
+ main_hop, *rest_hops = str(path).split("::")
+ if is_local_path(main_hop):
+ return os.path.relpath(main_hop, start=start) if start else os.path.relpath(main_hop)
+ else:
+ return posixpath.relpath(main_hop, start=str(start).split("::")[0]) if start else os.path.relpath(main_hop)
+
+
+class _OverridableIOWrapper(io.RawIOBase):
+ def __init__(self, f):
+ self._overrides = {}
+ self.f = f
+
+ def __getattribute__(self, attr):
+ if attr == "_overrides":
+ return object.__getattribute__(self, attr)
+ elif attr in self._overrides:
+ return self._overrides[attr]
+ return getattr(self.f, attr)
+
+ def __setattr__(self, attr, value):
+ if attr == "_overrides":
+ object.__setattr__(self, attr, value)
+ else:
+ self._overrides[attr] = value
+
+
+def _add_retries_to_file_obj_read_method(file_obj):
+ read = file_obj.read
+ max_retries = config.STREAMING_READ_MAX_RETRIES
+
+ def read_with_retries(*args, **kwargs):
+ disconnect_err = None
+ for retry in range(1, max_retries + 1):
+ try:
+ out = read(*args, **kwargs)
+ break
+ except CONNECTION_ERRORS_TO_RETRY as err:
+ disconnect_err = err
+ logger.warning(
+ f"Got disconnected from remote data host. Retrying in {config.STREAMING_READ_RETRY_INTERVAL}sec [{retry}/{max_retries}]"
+ )
+ time.sleep(config.STREAMING_READ_RETRY_INTERVAL)
+ except huggingface_hub.errors.HfHubHTTPError as err:
+ if err.response is not None and err.response.status_code == SERVER_UNAVAILABLE_CODE:
+ disconnect_err = err
+ logger.warning(
+ f"Got disconnected from remote data host. Retrying in {config.STREAMING_READ_SERVER_UNAVAILABLE_RETRY_INTERVAL}sec [{retry}/{max_retries}]"
+ )
+ time.sleep(config.STREAMING_READ_SERVER_UNAVAILABLE_RETRY_INTERVAL)
+ elif err.response is not None and err.response.status_code == RATE_LIMIT_CODE:
+ disconnect_err = err
+ logger.warning(str(err))
+ logger.warning(
+ f"Got disconnected from remote data host. Retrying in {config.STREAMING_READ_RATE_LIMIT_RETRY_INTERVAL}sec [{retry}/{max_retries}]"
+ )
+ time.sleep(config.STREAMING_READ_RATE_LIMIT_RETRY_INTERVAL)
+ else:
+ raise
+ else:
+ raise ConnectionError("Server Disconnected") from disconnect_err
+ return out
+
+ try:
+ file_obj.read = read_with_retries
+ except AttributeError: # read-only attribute
+ file_obj = _OverridableIOWrapper(file_obj)
+ file_obj.read = read_with_retries
+ return file_obj
+
+
+def _prepare_path_and_storage_options(
+ urlpath: str, download_config: Optional[DownloadConfig] = None
+) -> tuple[str, dict[str, dict[str, Any]]]:
+ prepared_urlpath = []
+ prepared_storage_options = {}
+ for hop in urlpath.split("::"):
+ hop, storage_options = _prepare_single_hop_path_and_storage_options(hop, download_config=download_config)
+ prepared_urlpath.append(hop)
+ prepared_storage_options.update(storage_options)
+ return "::".join(prepared_urlpath), storage_options
+
+
+def _prepare_single_hop_path_and_storage_options(
+ urlpath: str, download_config: Optional[DownloadConfig] = None
+) -> tuple[str, dict[str, dict[str, Any]]]:
+ """
+ Prepare the URL and the kwargs that must be passed to the HttpFileSystem or HfFileSystem
+
+ In particular it resolves google drive URLs
+ It also adds the authentication headers for the Hugging Face Hub, for both https:// and hf:// paths.
+
+ Storage options are formatted in the form {protocol: storage_options_for_protocol}
+ """
+ token = None if download_config is None else download_config.token
+ if urlpath.startswith(config.HF_ENDPOINT) and "/resolve/" in urlpath:
+ urlpath = "hf://" + urlpath[len(config.HF_ENDPOINT) + 1 :].replace("/resolve/", "@", 1)
+ protocol = urlpath.split("://")[0] if "://" in urlpath else "file"
+ if download_config is not None and protocol in download_config.storage_options:
+ storage_options = download_config.storage_options[protocol].copy()
+ elif download_config is not None and protocol not in download_config.storage_options:
+ storage_options = {
+ option_name: option_value
+ for option_name, option_value in download_config.storage_options.items()
+ if option_name not in fsspec.available_protocols()
+ }
+ else:
+ storage_options = {}
+ if protocol in {"http", "https"}:
+ client_kwargs = storage_options.pop("client_kwargs", {})
+ storage_options["client_kwargs"] = {"trust_env": True, **client_kwargs} # Enable reading proxy env variables
+ if "drive.google.com" in urlpath:
+ response = get_session().head(urlpath, timeout=10)
+ for k, v in response.cookies.items():
+ if k.startswith("download_warning"):
+ urlpath += "&confirm=" + v
+ cookies = response.cookies
+ storage_options = {"cookies": cookies, **storage_options}
+ # Fix Google Drive URL to avoid Virus scan warning
+ if "confirm=" not in urlpath:
+ urlpath += "&confirm=t"
+ if urlpath.startswith("https://raw.githubusercontent.com/"):
+ # Workaround for served data with gzip content-encoding: https://github.com/fsspec/filesystem_spec/issues/389
+ headers = storage_options.pop("headers", {})
+ storage_options["headers"] = {"Accept-Encoding": "identity", **headers}
+ elif protocol == "hf":
+ storage_options = {
+ "endpoint": config.HF_ENDPOINT,
+ "token": token,
+ **storage_options,
+ }
+ if storage_options:
+ storage_options = {protocol: storage_options}
+ return urlpath, storage_options
+
+
+def xopen(file: str, mode="r", *args, download_config: Optional[DownloadConfig] = None, **kwargs):
+ """Extend `open` function to support remote files using `fsspec`.
+
+ It also has a retry mechanism in case connection fails.
+ The `args` and `kwargs` are passed to `fsspec.open`, except `token` which is used for queries to private repos on huggingface.co
+
+ Args:
+ file (`str`): Path name of the file to be opened.
+ mode (`str`, *optional*, default "r"): Mode in which the file is opened.
+ *args: Arguments to be passed to `fsspec.open`.
+ download_config : mainly use token or storage_options to support different platforms and auth types.
+ **kwargs: Keyword arguments to be passed to `fsspec.open`.
+
+ Returns:
+ file object
+ """
+ # This works as well for `xopen(str(Path(...)))`
+ file_str = _as_str(file)
+ main_hop, *rest_hops = file_str.split("::")
+ if is_local_path(main_hop):
+ # ignore fsspec-specific kwargs
+ kwargs.pop("block_size", None)
+ return open(main_hop, mode, *args, **kwargs)
+ # add headers and cookies for authentication on the HF Hub and for Google Drive
+ file, storage_options = _prepare_path_and_storage_options(file_str, download_config=download_config)
+ kwargs = {**kwargs, **(storage_options or {})}
+
+ max_retries = config.STREAMING_OPEN_MAX_RETRIES
+
+ disconnect_err = None
+ for retry in range(1, max_retries + 1):
+ try:
+ fs, fs_token, paths = fsspec.get_fs_token_paths(
+ file,
+ mode,
+ storage_options=kwargs,
+ )
+ file_obj = fs.open(paths[0], mode)
+ if hasattr(fs, "of") and hasattr(fs.of, "__exit__"):
+ file_obj._fs = fs # keep a reference or the fs might close the file on gc
+ break
+ except CONNECTION_ERRORS_TO_RETRY as err:
+ disconnect_err = err
+ logger.warning(
+ f"Failed to connect to remote data host. Retrying in {config.STREAMING_OPEN_RETRY_INTERVAL}sec [{retry}/{max_retries}]"
+ )
+ time.sleep(config.STREAMING_OPEN_RETRY_INTERVAL)
+ except ValueError as e:
+ if str(e) == "Cannot seek streaming HTTP file":
+ raise NonStreamableDatasetError(
+ "Streaming is not possible for this dataset because data host server doesn't support HTTP range "
+ "requests. You can still load this dataset in non-streaming mode by passing `streaming=False` (default)"
+ ) from e
+ else:
+ raise
+ except FileNotFoundError:
+ if file.startswith(config.HF_ENDPOINT):
+ raise FileNotFoundError(
+ file + "\nIf the repo is private or gated, make sure to log in with `huggingface-cli login`."
+ ) from None
+ else:
+ raise
+ else:
+ raise ConnectionError("Server Disconnected") from disconnect_err
+ file_obj = _add_retries_to_file_obj_read_method(file_obj)
+ return file_obj
+
+
+def xlistdir(path: str, download_config: Optional[DownloadConfig] = None) -> list[str]:
+ """Extend `os.listdir` function to support remote files.
+
+ Args:
+ path (`str`): URL path.
+ download_config : mainly use token or storage_options to support different platforms and auth types.
+
+ Returns:
+ `list` of `str`
+ """
+ main_hop, *rest_hops = _as_str(path).split("::")
+ if is_local_path(main_hop):
+ return os.listdir(path)
+ else:
+ # globbing inside a zip in a private repo requires authentication
+ path, storage_options = _prepare_path_and_storage_options(path, download_config=download_config)
+ main_hop, *rest_hops = path.split("::")
+ fs, *_ = url_to_fs(path, **storage_options)
+ inner_path = main_hop.split("://")[-1]
+ if inner_path.strip("/") and not fs.isdir(inner_path):
+ raise FileNotFoundError(f"Directory doesn't exist: {path}")
+ paths = fs.listdir(inner_path, detail=False)
+ return [os.path.basename(path.rstrip("/")) for path in paths]
+
+
+def xglob(urlpath, *, recursive=False, download_config: Optional[DownloadConfig] = None):
+ """Extend `glob.glob` function to support remote files.
+
+ Args:
+ urlpath (`str`): URL path with shell-style wildcard patterns.
+ recursive (`bool`, default `False`): Whether to match the "**" pattern recursively to zero or more
+ directories or subdirectories.
+ download_config : mainly use token or storage_options to support different platforms and auth types.
+
+ Returns:
+ `list` of `str`
+ """
+ main_hop, *rest_hops = _as_str(urlpath).split("::")
+ if is_local_path(main_hop):
+ return glob.glob(main_hop, recursive=recursive)
+ else:
+ # globbing inside a zip in a private repo requires authentication
+ urlpath, storage_options = _prepare_path_and_storage_options(urlpath, download_config=download_config)
+ main_hop, *rest_hops = urlpath.split("::")
+ fs, *_ = url_to_fs(urlpath, **storage_options)
+ inner_path = main_hop.split("://")[1]
+ globbed_paths = fs.glob(inner_path)
+ protocol = fs.protocol if isinstance(fs.protocol, str) else fs.protocol[-1]
+ return ["::".join([f"{protocol}://{globbed_path}"] + rest_hops) for globbed_path in globbed_paths]
+
+
+def xwalk(urlpath, download_config: Optional[DownloadConfig] = None, **kwargs):
+ """Extend `os.walk` function to support remote files.
+
+ Args:
+ urlpath (`str`): URL root path.
+ download_config : mainly use token or storage_options to support different platforms and auth types.
+ **kwargs: Additional keyword arguments forwarded to the underlying filesystem.
+
+
+ Yields:
+ `tuple`: 3-tuple (dirpath, dirnames, filenames).
+ """
+ main_hop, *rest_hops = _as_str(urlpath).split("::")
+ if is_local_path(main_hop):
+ yield from os.walk(main_hop, **kwargs)
+ else:
+ # walking inside a zip in a private repo requires authentication
+ urlpath, storage_options = _prepare_path_and_storage_options(urlpath, download_config=download_config)
+ main_hop, *rest_hops = urlpath.split("::")
+ fs, *_ = url_to_fs(urlpath, **storage_options)
+ inner_path = main_hop.split("://")[-1]
+ if inner_path.strip("/") and not fs.isdir(inner_path):
+ return []
+ protocol = fs.protocol if isinstance(fs.protocol, str) else fs.protocol[-1]
+ for dirpath, dirnames, filenames in fs.walk(inner_path, **kwargs):
+ yield "::".join([f"{protocol}://{dirpath}"] + rest_hops), dirnames, filenames
+
+
+class xPath(type(Path())):
+ """Extension of `pathlib.Path` to support both local paths and remote URLs."""
+
+ def __str__(self):
+ path_str = super().__str__()
+ main_hop, *rest_hops = path_str.split("::")
+ if is_local_path(main_hop):
+ return main_hop
+ path_as_posix = path_str.replace("\\", "/")
+ path_as_posix = SINGLE_SLASH_AFTER_PROTOCOL_PATTERN.sub("://", path_as_posix)
+ path_as_posix += "//" if path_as_posix.endswith(":") else "" # Add slashes to root of the protocol
+ return path_as_posix
+
+ def exists(self, download_config: Optional[DownloadConfig] = None):
+ """Extend `pathlib.Path.exists` method to support both local and remote files.
+
+ Args:
+ download_config : mainly use token or storage_options to support different platforms and auth types.
+
+ Returns:
+ `bool`
+ """
+ return xexists(str(self), download_config=download_config)
+
+ def glob(self, pattern, download_config: Optional[DownloadConfig] = None):
+ """Glob function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs.
+
+ Args:
+ pattern (`str`): Pattern that resulting paths must match.
+ download_config : mainly use token or storage_options to support different platforms and auth types.
+
+ Yields:
+ [`xPath`]
+ """
+ posix_path = self.as_posix()
+ main_hop, *rest_hops = posix_path.split("::")
+ if is_local_path(main_hop):
+ yield from Path(main_hop).glob(pattern)
+ else:
+ # globbing inside a zip in a private repo requires authentication
+ if rest_hops:
+ urlpath = rest_hops[0]
+ urlpath, storage_options = _prepare_path_and_storage_options(urlpath, download_config=download_config)
+ storage_options = {urlpath.split("://")[0]: storage_options}
+ posix_path = "::".join([main_hop, urlpath, *rest_hops[1:]])
+ else:
+ storage_options = None
+ fs, *_ = url_to_fs(xjoin(posix_path, pattern), **(storage_options or {}))
+ globbed_paths = fs.glob(xjoin(main_hop, pattern))
+ for globbed_path in globbed_paths:
+ yield type(self)("::".join([f"{fs.protocol}://{globbed_path}"] + rest_hops))
+
+ def rglob(self, pattern, **kwargs):
+ """Rglob function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs.
+
+ Args:
+ pattern (`str`): Pattern that resulting paths must match.
+
+ Yields:
+ [`xPath`]
+ """
+ return self.glob("**/" + pattern, **kwargs)
+
+ @property
+ def parent(self) -> "xPath":
+ """Name function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs.
+
+ Returns:
+ [`xPath`]
+ """
+ return type(self)(xdirname(self.as_posix()))
+
+ @property
+ def name(self) -> str:
+ """Name function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs.
+
+ Returns:
+ `str`
+ """
+ return PurePosixPath(self.as_posix().split("::")[0]).name
+
+ @property
+ def stem(self) -> str:
+ """Stem function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs.
+
+ Returns:
+ `str`
+ """
+ return PurePosixPath(self.as_posix().split("::")[0]).stem
+
+ @property
+ def suffix(self) -> str:
+ """Suffix function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs.
+
+ Returns:
+ `str`
+ """
+ return PurePosixPath(self.as_posix().split("::")[0]).suffix
+
+ def open(self, *args, **kwargs):
+ """Extend :func:`xopen` to support argument of type :obj:`~pathlib.Path`.
+
+ Args:
+ **args: Arguments passed to :func:`fsspec.open`.
+ **kwargs: Keyword arguments passed to :func:`fsspec.open`.
+
+ Returns:
+ `io.FileIO`: File-like object.
+ """
+ return xopen(str(self), *args, **kwargs)
+
+ def joinpath(self, *p: tuple[str, ...]) -> "xPath":
+ """Extend :func:`xjoin` to support argument of type :obj:`~pathlib.Path`.
+
+ Args:
+ *p (`tuple` of `str`): Other path components.
+
+ Returns:
+ [`xPath`]
+ """
+ return type(self)(xjoin(self.as_posix(), *p))
+
+ def __truediv__(self, p: str) -> "xPath":
+ return self.joinpath(p)
+
+ def with_suffix(self, suffix):
+ main_hop, *rest_hops = str(self).split("::")
+ if is_local_path(main_hop):
+ return type(self)(str(super().with_suffix(suffix)))
+ return type(self)("::".join([type(self)(PurePosixPath(main_hop).with_suffix(suffix)).as_posix()] + rest_hops))
+
+
+def _as_str(path: Union[str, Path, xPath]):
+ return str(path) if isinstance(path, xPath) else str(xPath(str(path)))
+
+
+def xgzip_open(filepath_or_buffer, *args, download_config: Optional[DownloadConfig] = None, **kwargs):
+ import gzip
+
+ if hasattr(filepath_or_buffer, "read"):
+ return gzip.open(filepath_or_buffer, *args, **kwargs)
+ else:
+ filepath_or_buffer = str(filepath_or_buffer)
+ return gzip.open(xopen(filepath_or_buffer, "rb", download_config=download_config), *args, **kwargs)
+
+
+def xnumpy_load(filepath_or_buffer, *args, download_config: Optional[DownloadConfig] = None, **kwargs):
+ import numpy as np
+
+ if hasattr(filepath_or_buffer, "read"):
+ return np.load(filepath_or_buffer, *args, **kwargs)
+ else:
+ filepath_or_buffer = str(filepath_or_buffer)
+ return np.load(xopen(filepath_or_buffer, "rb", download_config=download_config), *args, **kwargs)
+
+
+def xpandas_read_csv(filepath_or_buffer, download_config: Optional[DownloadConfig] = None, **kwargs):
+ import pandas as pd
+
+ if hasattr(filepath_or_buffer, "read"):
+ return pd.read_csv(filepath_or_buffer, **kwargs)
+ else:
+ filepath_or_buffer = str(filepath_or_buffer)
+ if kwargs.get("compression", "infer") == "infer":
+ kwargs["compression"] = _get_extraction_protocol(filepath_or_buffer, download_config=download_config)
+ return pd.read_csv(xopen(filepath_or_buffer, "rb", download_config=download_config), **kwargs)
+
+
+def xpandas_read_excel(filepath_or_buffer, download_config: Optional[DownloadConfig] = None, **kwargs):
+ import pandas as pd
+
+ if hasattr(filepath_or_buffer, "read"):
+ try:
+ return pd.read_excel(filepath_or_buffer, **kwargs)
+ except ValueError: # Cannot seek streaming HTTP file
+ return pd.read_excel(BytesIO(filepath_or_buffer.read()), **kwargs)
+ else:
+ filepath_or_buffer = str(filepath_or_buffer)
+ try:
+ return pd.read_excel(xopen(filepath_or_buffer, "rb", download_config=download_config), **kwargs)
+ except ValueError: # Cannot seek streaming HTTP file
+ return pd.read_excel(
+ BytesIO(xopen(filepath_or_buffer, "rb", download_config=download_config).read()), **kwargs
+ )
+
+
+def xpyarrow_parquet_read_table(filepath_or_buffer, download_config: Optional[DownloadConfig] = None, **kwargs):
+ import pyarrow.parquet as pq
+
+ if hasattr(filepath_or_buffer, "read"):
+ return pq.read_table(filepath_or_buffer, **kwargs)
+ else:
+ filepath_or_buffer = str(filepath_or_buffer)
+ return pq.read_table(xopen(filepath_or_buffer, mode="rb", download_config=download_config), **kwargs)
+
+
+def xsio_loadmat(filepath_or_buffer, download_config: Optional[DownloadConfig] = None, **kwargs):
+ import scipy.io as sio
+
+ if hasattr(filepath_or_buffer, "read"):
+ return sio.loadmat(filepath_or_buffer, **kwargs)
+ else:
+ return sio.loadmat(xopen(filepath_or_buffer, "rb", download_config=download_config), **kwargs)
+
+
+def xet_parse(source, parser=None, download_config: Optional[DownloadConfig] = None):
+ """Extend `xml.etree.ElementTree.parse` function to support remote files.
+
+ Args:
+ source: File path or file object.
+ parser (`XMLParser`, *optional*, default `XMLParser`): Parser instance.
+ download_config : mainly use token or storage_options to support different platforms and auth types.
+
+ Returns:
+ `xml.etree.ElementTree.Element`: Root element of the given source document.
+ """
+ if hasattr(source, "read"):
+ return ET.parse(source, parser=parser)
+ else:
+ with xopen(source, "rb", download_config=download_config) as f:
+ return ET.parse(f, parser=parser)
+
+
+def xxml_dom_minidom_parse(filename_or_file, download_config: Optional[DownloadConfig] = None, **kwargs):
+ """Extend `xml.dom.minidom.parse` function to support remote files.
+
+ Args:
+ filename_or_file (`str` or file): File path or file object.
+ download_config : mainly use token or storage_options to support different platforms and auth types.
+ **kwargs (optional): Additional keyword arguments passed to `xml.dom.minidom.parse`.
+
+ Returns:
+ :obj:`xml.dom.minidom.Document`: Parsed document.
+ """
+ if hasattr(filename_or_file, "read"):
+ return xml.dom.minidom.parse(filename_or_file, **kwargs)
+ else:
+ with xopen(filename_or_file, "rb", download_config=download_config) as f:
+ return xml.dom.minidom.parse(f, **kwargs)
+
+
+class ArchiveIterable(TrackedIterableFromGenerator):
+ """An iterable of (path, fileobj) from a TAR archive, used by `iter_archive`"""
+
+ @staticmethod
+ def _iter_tar(f):
+ stream = tarfile.open(fileobj=f, mode="r|*")
+ for tarinfo in stream:
+ file_path = tarinfo.name
+ if not tarinfo.isreg():
+ continue
+ if file_path is None:
+ continue
+ if os.path.basename(file_path).startswith((".", "__")):
+ # skipping hidden files
+ continue
+ file_obj = stream.extractfile(tarinfo)
+ yield file_path, file_obj
+ stream.members = []
+ del stream
+
+ @staticmethod
+ def _iter_zip(f):
+ zipf = zipfile.ZipFile(f)
+ for member in zipf.infolist():
+ file_path = member.filename
+ if member.is_dir():
+ continue
+ if file_path is None:
+ continue
+ if os.path.basename(file_path).startswith((".", "__")):
+ # skipping hidden files
+ continue
+ file_obj = zipf.open(member)
+ yield file_path, file_obj
+
+ @classmethod
+ def _iter_from_fileobj(cls, f) -> Generator[tuple, None, None]:
+ compression = _get_extraction_protocol_with_magic_number(f)
+ if compression == "zip":
+ yield from cls._iter_zip(f)
+ else:
+ yield from cls._iter_tar(f)
+
+ @classmethod
+ def _iter_from_urlpath(
+ cls, urlpath: str, download_config: Optional[DownloadConfig] = None
+ ) -> Generator[tuple, None, None]:
+ compression = _get_extraction_protocol(urlpath, download_config=download_config)
+ # Set block_size=0 to get faster streaming
+ # (e.g. for hf:// and https:// it uses streaming Requests file-like instances)
+ with xopen(urlpath, "rb", download_config=download_config, block_size=0) as f:
+ if compression == "zip":
+ yield from cls._iter_zip(f)
+ else:
+ yield from cls._iter_tar(f)
+
+ @classmethod
+ def from_buf(cls, fileobj) -> "ArchiveIterable":
+ return cls(cls._iter_from_fileobj, fileobj)
+
+ @classmethod
+ def from_urlpath(cls, urlpath_or_buf, download_config: Optional[DownloadConfig] = None) -> "ArchiveIterable":
+ return cls(cls._iter_from_urlpath, urlpath_or_buf, download_config)
+
+
+class FilesIterable(TrackedIterableFromGenerator):
+ """An iterable of paths from a list of directories or files"""
+
+ @classmethod
+ def _iter_from_urlpaths(
+ cls, urlpaths: Union[str, list[str]], download_config: Optional[DownloadConfig] = None
+ ) -> Generator[str, None, None]:
+ if not isinstance(urlpaths, list):
+ urlpaths = [urlpaths]
+ for urlpath in urlpaths:
+ if xisfile(urlpath, download_config=download_config):
+ yield urlpath
+ elif xisdir(urlpath, download_config=download_config):
+ for dirpath, dirnames, filenames in xwalk(urlpath, download_config=download_config):
+ # in-place modification to prune the search
+ dirnames[:] = sorted([dirname for dirname in dirnames if not dirname.startswith((".", "__"))])
+ if xbasename(dirpath).startswith((".", "__")):
+ # skipping hidden directories
+ continue
+ for filename in sorted(filenames):
+ if filename.startswith((".", "__")):
+ # skipping hidden files
+ continue
+ yield xjoin(dirpath, filename)
+ else:
+ raise FileNotFoundError(urlpath)
+
+ @classmethod
+ def from_urlpaths(cls, urlpaths, download_config: Optional[DownloadConfig] = None) -> "FilesIterable":
+ return cls(cls._iter_from_urlpaths, urlpaths, download_config)
diff --git a/venv/Lib/site-packages/datasets/utils/filelock.py b/venv/Lib/site-packages/datasets/utils/filelock.py
new file mode 100644
index 0000000000000000000000000000000000000000..df0728efe644d8eb32f0e578a85e39ba366e9743
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/filelock.py
@@ -0,0 +1,11 @@
+# deprecated, please use the `filelock` package instead
+
+from filelock import ( # noqa: F401 # imported for backward compatibility TODO: remove in 3.0.0
+ BaseFileLock,
+ SoftFileLock,
+ Timeout,
+ UnixFileLock,
+ WindowsFileLock,
+)
+
+from ._filelock import FileLock # noqa: F401 # imported for backward compatibility. TODO: remove in 3.0.0
diff --git a/venv/Lib/site-packages/datasets/utils/hub.py b/venv/Lib/site-packages/datasets/utils/hub.py
new file mode 100644
index 0000000000000000000000000000000000000000..6d784333b23328c113ec7be444d0a77410f1b857
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/hub.py
@@ -0,0 +1,6 @@
+from functools import partial
+
+from huggingface_hub import hf_hub_url
+
+
+hf_dataset_url = partial(hf_hub_url, repo_type="dataset")
diff --git a/venv/Lib/site-packages/datasets/utils/info_utils.py b/venv/Lib/site-packages/datasets/utils/info_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..c3e246f850ffd58c3b572c5a5038b88e863be4fb
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/info_utils.py
@@ -0,0 +1,105 @@
+import enum
+import os
+from typing import Optional
+
+from huggingface_hub.utils import insecure_hashlib
+
+from .. import config
+from ..exceptions import (
+ ExpectedMoreDownloadedFilesError,
+ ExpectedMoreSplitsError,
+ NonMatchingChecksumError,
+ NonMatchingSplitsSizesError,
+ UnexpectedDownloadedFileError,
+ UnexpectedSplitsError,
+)
+from .logging import get_logger
+
+
+logger = get_logger(__name__)
+
+
+class VerificationMode(enum.Enum):
+ """`Enum` that specifies which verification checks to run.
+
+ The default mode is `BASIC_CHECKS`, which will perform only rudimentary checks to avoid slowdowns
+ when generating/downloading a dataset for the first time.
+
+ The verification modes:
+
+ | | Verification checks |
+ |---------------------------|------------------------------------------------------------------------------ |
+ | `ALL_CHECKS` | Split checks and validity (number of files, checksums) of downloaded files |
+ | `BASIC_CHECKS` (default) | Same as `ALL_CHECKS` but without checking downloaded files |
+ | `NO_CHECKS` | None |
+
+ """
+
+ ALL_CHECKS = "all_checks"
+ BASIC_CHECKS = "basic_checks"
+ NO_CHECKS = "no_checks"
+
+
+def verify_checksums(expected_checksums: Optional[dict], recorded_checksums: dict, verification_name=None):
+ if expected_checksums is None:
+ logger.info("Unable to verify checksums.")
+ return
+ if len(set(expected_checksums) - set(recorded_checksums)) > 0:
+ raise ExpectedMoreDownloadedFilesError(str(set(expected_checksums) - set(recorded_checksums)))
+ if len(set(recorded_checksums) - set(expected_checksums)) > 0:
+ raise UnexpectedDownloadedFileError(str(set(recorded_checksums) - set(expected_checksums)))
+ bad_urls = [url for url in expected_checksums if expected_checksums[url] != recorded_checksums[url]]
+ for_verification_name = " for " + verification_name if verification_name is not None else ""
+ if len(bad_urls) > 0:
+ raise NonMatchingChecksumError(
+ f"Checksums didn't match{for_verification_name}:\n"
+ f"{bad_urls}\n"
+ "Set `verification_mode='no_checks'` to skip checksums verification and ignore this error"
+ )
+ logger.info("All the checksums matched successfully" + for_verification_name)
+
+
+def verify_splits(expected_splits: Optional[dict], recorded_splits: dict):
+ if expected_splits is None:
+ logger.info("Unable to verify splits sizes.")
+ return
+ if len(set(expected_splits) - set(recorded_splits)) > 0:
+ raise ExpectedMoreSplitsError(str(set(expected_splits) - set(recorded_splits)))
+ if len(set(recorded_splits) - set(expected_splits)) > 0:
+ raise UnexpectedSplitsError(str(set(recorded_splits) - set(expected_splits)))
+ bad_splits = [
+ {"expected": expected_splits[name], "recorded": recorded_splits[name]}
+ for name in expected_splits
+ if expected_splits[name].num_examples != recorded_splits[name].num_examples
+ ]
+ if len(bad_splits) > 0:
+ raise NonMatchingSplitsSizesError(str(bad_splits))
+ logger.info("All the splits matched successfully.")
+
+
+def get_size_checksum_dict(path: str, record_checksum: bool = False) -> dict:
+ """Compute the file size and the sha256 checksum of a file"""
+ if record_checksum:
+ m = insecure_hashlib.sha256()
+ with open(path, "rb") as f:
+ for chunk in iter(lambda: f.read(1 << 20), b""):
+ m.update(chunk)
+ checksum = m.hexdigest()
+ else:
+ checksum = None
+ return {"num_bytes": os.path.getsize(path), "checksum": checksum}
+
+
+def is_small_dataset(dataset_size):
+ """Check if `dataset_size` is smaller than `config.IN_MEMORY_MAX_SIZE`.
+
+ Args:
+ dataset_size (int): Dataset size in bytes.
+
+ Returns:
+ bool: Whether `dataset_size` is smaller than `config.IN_MEMORY_MAX_SIZE`.
+ """
+ if dataset_size and config.IN_MEMORY_MAX_SIZE:
+ return dataset_size < config.IN_MEMORY_MAX_SIZE
+ else:
+ return False
diff --git a/venv/Lib/site-packages/datasets/utils/json.py b/venv/Lib/site-packages/datasets/utils/json.py
new file mode 100644
index 0000000000000000000000000000000000000000..c1fb4e321da6ac51dd7b93527af6a34ff3fa83c8
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/json.py
@@ -0,0 +1,139 @@
+from typing import TYPE_CHECKING, Any
+
+import pandas as pd
+
+
+if TYPE_CHECKING:
+ from ..features.features import FeatureType
+
+
+def ujson_dumps(*args, **kwargs):
+ try:
+ return pd.io.json.ujson_dumps(*args, **kwargs)
+ except AttributeError:
+ # Before pandas-2.2.0, ujson_dumps was renamed to dumps: import ujson_dumps as dumps
+ return pd.io.json.dumps(*args, **kwargs)
+
+
+def ujson_loads(*args, **kwargs):
+ try:
+ return pd.io.json.ujson_loads(*args, **kwargs)
+ except AttributeError:
+ # Before pandas-2.2.0, ujson_loads was renamed to loads: import ujson_loads as loads
+ return pd.io.json.loads(*args, **kwargs)
+
+
+def json_encode_field(example: Any, json_field_path: list[str]) -> Any:
+ if json_field_path:
+ field, *json_field_path = json_field_path
+ if example is None:
+ return None
+ elif field == 0:
+ return [json_encode_field(x, json_field_path) for x in example]
+ else:
+ return {**example, field: json_encode_field(example.get(field), json_field_path)}
+ elif example is None:
+ # keep missing values as real nulls instead of the JSON string "null"
+ return None
+ else:
+ try:
+ ujson_loads(example)
+ except Exception:
+ return ujson_dumps(example)
+ else:
+ return example
+
+
+def json_decode_field(example: Any, json_field_path: str) -> Any:
+ if json_field_path:
+ field, *json_field_path = json_field_path
+ if example is None:
+ return None
+ elif field == 0:
+ return [json_decode_field(x, json_field_path) for x in example]
+ else:
+ return {**example, field: json_decode_field(example.get(field), json_field_path)}
+ elif example is None:
+ return None
+ else:
+ try:
+ return ujson_loads(example)
+ except Exception:
+ return example
+
+
+def find_mixed_struct_types_field_paths(examples: list, allow_root=False) -> list[list[str]]:
+ mixed_struct_types_field_paths = []
+ examples = [example for example in examples if example is not None]
+ if not examples:
+ return []
+ paths_and_content_to_check = [([], examples)]
+ while paths_and_content_to_check:
+ path, content = paths_and_content_to_check.pop(0)
+ if all(isinstance(x, dict) for x in content):
+ if (allow_root or path) and (any(set(x) != set(content[0]) for x in content) or not content[0]):
+ mixed_struct_types_field_paths.append(path)
+ else:
+ for subfield in {field for x in content for field in x}:
+ examples = [x[subfield] for x in content if subfield in x and x[subfield] is not None]
+ if not examples:
+ continue
+ paths_and_content_to_check.append((path + [subfield], examples))
+ elif all(isinstance(x, list) for x in content):
+ examples = [x for sublist in content for x in sublist if x is not None]
+ if not examples:
+ continue
+ paths_and_content_to_check.append((path + [0], examples))
+ elif any(isinstance(x, (dict, list)) for x in content):
+ mixed_struct_types_field_paths.append(path)
+ return mixed_struct_types_field_paths
+
+
+def get_json_field_path_from_pyarrow_json_error(err_str: str) -> list[str]:
+ # e.g. json_field_path_str = "col/subfield_containing_a_list/[]/subsubfield_in_item_in_the_list"
+ json_field_path_str = err_str.split("Column(", 1)[1].rsplit(") changed from", 1)[0].strip("/")
+ # e.g. json_field_path = ["col", "subfield_containing_a_list", 0, "subsubfield_in_item_in_the_list"]
+ json_field_path = [0 if seg == "[]" else seg for seg in json_field_path_str.split("/")]
+ return json_field_path
+
+
+def insert_json_field_path(json_field_paths: list[list[str]], json_field_path: list[str]) -> None:
+ # Add to list of json_field_paths and check if other share a common path
+ for i in range(len(json_field_paths)):
+ if json_field_paths[i][: len(json_field_path)] == json_field_path:
+ json_field_paths[i] = json_field_path
+ break
+ else:
+ json_field_paths.append(json_field_path)
+
+
+def json_encode_fields_in_json_lines(original_batch: bytes, json_field_paths: list[list[str]]) -> bytes:
+ examples = [ujson_loads(line) for line in original_batch.splitlines()]
+ for json_field_path in json_field_paths:
+ examples = [json_encode_field(example, json_field_path) for example in examples]
+ batch = "\n".join([ujson_dumps(example) for example in examples]).encode()
+ return batch
+
+
+def get_json_field_paths_from_feature(feature: "FeatureType") -> list[list[str]]:
+ from datasets.features.features import Json, _visit_with_path
+
+ json_field_paths = []
+
+ def get_json_type_path(_feature, feature_path):
+ if isinstance(_feature, Json):
+ json_field_paths.append(feature_path)
+ return _feature
+
+ _visit_with_path(feature, get_json_type_path)
+ return json_field_paths
+
+
+def set_json_types_in_feature(feature: "FeatureType", json_field_paths: list[list[str]]) -> None:
+ from datasets.features.features import Json, _visit_with_path
+
+ def set_json_type(feature, feature_path):
+ return Json() if feature_path in json_field_paths else feature
+
+ feature = _visit_with_path(feature, set_json_type)
+ return feature
diff --git a/venv/Lib/site-packages/datasets/utils/logging.py b/venv/Lib/site-packages/datasets/utils/logging.py
new file mode 100644
index 0000000000000000000000000000000000000000..851c1a2b73d2f411dbd180e16e37daf91181e13c
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/logging.py
@@ -0,0 +1,175 @@
+# Copyright 2020 Optuna, Hugging Face
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Logging utilities."""
+
+import logging
+import os
+from logging import (
+ CRITICAL, # NOQA
+ DEBUG, # NOQA
+ ERROR, # NOQA
+ FATAL, # NOQA
+ INFO, # NOQA
+ NOTSET, # NOQA
+ WARN, # NOQA
+ WARNING, # NOQA
+)
+from typing import Optional
+
+from .tqdm import ( # noqa: F401 # imported for backward compatibility
+ disable_progress_bar,
+ enable_progress_bar,
+ is_progress_bar_enabled,
+ tqdm,
+)
+
+
+log_levels = {
+ "debug": logging.DEBUG,
+ "info": logging.INFO,
+ "warning": logging.WARNING,
+ "error": logging.ERROR,
+ "critical": logging.CRITICAL,
+}
+
+_default_log_level = logging.WARNING
+
+
+def _get_default_logging_level():
+ """
+ If DATASETS_VERBOSITY env var is set to one of the valid choices return that as the new default level.
+ If it is not - fall back to ``_default_log_level``
+ """
+ env_level_str = os.getenv("DATASETS_VERBOSITY", None)
+ if env_level_str:
+ if env_level_str in log_levels:
+ return log_levels[env_level_str]
+ else:
+ logging.getLogger().warning(
+ f"Unknown option DATASETS_VERBOSITY={env_level_str}, has to be one of: {', '.join(log_levels.keys())}"
+ )
+ return _default_log_level
+
+
+def _get_library_name() -> str:
+ return __name__.split(".")[0]
+
+
+def _get_library_root_logger() -> logging.Logger:
+ return logging.getLogger(_get_library_name())
+
+
+def _configure_library_root_logger() -> None:
+ # Apply our default configuration to the library root logger.
+ library_root_logger = _get_library_root_logger()
+ library_root_logger.addHandler(logging.StreamHandler())
+ library_root_logger.setLevel(_get_default_logging_level())
+
+
+def _reset_library_root_logger() -> None:
+ library_root_logger = _get_library_root_logger()
+ library_root_logger.setLevel(logging.NOTSET)
+
+
+def get_logger(name: Optional[str] = None) -> logging.Logger:
+ """Return a logger with the specified name.
+ This function can be used in dataset builders.
+ """
+ if name is None:
+ name = _get_library_name()
+ return logging.getLogger(name)
+
+
+def get_verbosity() -> int:
+ """Return the current level for the HuggingFace datasets library's root logger.
+ Returns:
+ Logging level, e.g., `datasets.logging.DEBUG` and `datasets.logging.INFO`.
+
+ > [!TIP]
+ > HuggingFace datasets library has following logging levels:
+ > - `datasets.logging.CRITICAL`, `datasets.logging.FATAL`
+ > - `datasets.logging.ERROR`
+ > - `datasets.logging.WARNING`, `datasets.logging.WARN`
+ > - `datasets.logging.INFO`
+ > - `datasets.logging.DEBUG`
+ """
+ return _get_library_root_logger().getEffectiveLevel()
+
+
+def set_verbosity(verbosity: int) -> None:
+ """Set the level for the Hugging Face Datasets library's root logger.
+ Args:
+ verbosity:
+ Logging level, e.g., `datasets.logging.DEBUG` and `datasets.logging.INFO`.
+ """
+ _get_library_root_logger().setLevel(verbosity)
+
+
+def set_verbosity_info():
+ """Set the level for the Hugging Face datasets library's root logger to `INFO`.
+
+ This will display most of the logging information and tqdm bars.
+
+ Shortcut to `datasets.logging.set_verbosity(datasets.logging.INFO)`.
+ """
+ return set_verbosity(INFO)
+
+
+def set_verbosity_warning():
+ """Set the level for the Hugging Face datasets library's root logger to `WARNING`.
+
+ This will display only the warning and errors logging information and tqdm bars.
+
+ Shortcut to `datasets.logging.set_verbosity(datasets.logging.WARNING)`.
+ """
+ return set_verbosity(WARNING)
+
+
+def set_verbosity_debug():
+ """Set the level for the Hugging Face datasets library's root logger to `DEBUG`.
+
+ This will display all the logging information and tqdm bars.
+
+ Shortcut to `datasets.logging.set_verbosity(datasets.logging.DEBUG)`.
+ """
+ return set_verbosity(DEBUG)
+
+
+def set_verbosity_error():
+ """Set the level for the Hugging Face datasets library's root logger to `ERROR`.
+
+ This will display only the errors logging information and tqdm bars.
+
+ Shortcut to `datasets.logging.set_verbosity(datasets.logging.ERROR)`.
+ """
+ return set_verbosity(ERROR)
+
+
+def disable_propagation() -> None:
+ """Disable propagation of the library log outputs.
+ Note that log propagation is disabled by default.
+ """
+ _get_library_root_logger().propagate = False
+
+
+def enable_propagation() -> None:
+ """Enable propagation of the library log outputs.
+ Please disable the Hugging Face datasets library's default handler to prevent double logging if the root logger has
+ been configured.
+ """
+ _get_library_root_logger().propagate = True
+
+
+# Configure the library root logger at the module level (singleton-like)
+_configure_library_root_logger()
diff --git a/venv/Lib/site-packages/datasets/utils/metadata.py b/venv/Lib/site-packages/datasets/utils/metadata.py
new file mode 100644
index 0000000000000000000000000000000000000000..cceecf35d63f518fcb7a62c804b9679e5d0c144e
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/metadata.py
@@ -0,0 +1,233 @@
+import re
+import textwrap
+from collections import Counter
+from itertools import groupby
+from operator import itemgetter
+from typing import Any, ClassVar, Optional
+
+import yaml
+from huggingface_hub import DatasetCardData
+
+from ..config import METADATA_CONFIGS_FIELD
+from ..features import Features
+from ..info import DatasetInfo, DatasetInfosDict
+from ..naming import _split_re
+from ..utils.logging import get_logger
+
+
+logger = get_logger(__name__)
+
+
+class _NoDuplicateSafeLoader(yaml.SafeLoader):
+ def _check_no_duplicates_on_constructed_node(self, node):
+ keys = [self.constructed_objects[key_node] for key_node, _ in node.value]
+ keys = [tuple(key) if isinstance(key, list) else key for key in keys]
+ counter = Counter(keys)
+ duplicate_keys = [key for key in counter if counter[key] > 1]
+ if duplicate_keys:
+ raise TypeError(f"Got duplicate yaml keys: {duplicate_keys}")
+
+ def construct_mapping(self, node, deep=False):
+ mapping = super().construct_mapping(node, deep=deep)
+ self._check_no_duplicates_on_constructed_node(node)
+ return mapping
+
+
+def _split_yaml_from_readme(readme_content: str) -> tuple[Optional[str], str]:
+ full_content = list(readme_content.splitlines())
+ if full_content and full_content[0] == "---" and "---" in full_content[1:]:
+ sep_idx = full_content[1:].index("---") + 1
+ yamlblock = "\n".join(full_content[1:sep_idx])
+ return yamlblock, "\n".join(full_content[sep_idx + 1 :])
+
+ return None, "\n".join(full_content)
+
+
+class MetadataConfigs(dict[str, dict[str, Any]]):
+ """Should be in format {config_name: {**config_params}}."""
+
+ FIELD_NAME: ClassVar[str] = METADATA_CONFIGS_FIELD
+
+ @staticmethod
+ def _raise_if_data_files_field_not_valid(metadata_config: dict):
+ yaml_data_files = metadata_config.get("data_files")
+ if yaml_data_files is not None:
+ yaml_error_message = textwrap.dedent(
+ f"""
+ Expected data_files in YAML to be either a string or a list of strings
+ or a list of dicts with two keys: 'split' and 'path', but got {yaml_data_files}
+ Examples of data_files in YAML:
+
+ data_files: data.csv
+
+ data_files: data/*.png
+
+ data_files:
+ - part0/*
+ - part1/*
+
+ data_files:
+ - split: train
+ path: train/*
+ - split: test
+ path: test/*
+
+ data_files:
+ - split: train
+ path:
+ - train/part1/*
+ - train/part2/*
+ - split: test
+ path: test/*
+
+ PS: some symbols like dashes '-' are not allowed in split names
+ """
+ )
+ if not isinstance(yaml_data_files, (list, str)):
+ raise ValueError(yaml_error_message)
+ if isinstance(yaml_data_files, list):
+ for yaml_data_files_item in yaml_data_files:
+ if (
+ not isinstance(yaml_data_files_item, (str, dict))
+ or isinstance(yaml_data_files_item, dict)
+ and not (
+ len(yaml_data_files_item) == 2
+ and "split" in yaml_data_files_item
+ and re.match(_split_re, yaml_data_files_item["split"])
+ and isinstance(yaml_data_files_item.get("path"), (str, list))
+ )
+ ):
+ raise ValueError(yaml_error_message)
+
+ @classmethod
+ def _from_exported_parquet_files_and_dataset_infos(
+ cls,
+ parquet_commit_hash: str,
+ exported_parquet_files: list[dict[str, Any]],
+ dataset_infos: DatasetInfosDict,
+ ) -> "MetadataConfigs":
+ metadata_configs = {
+ config_name: {
+ "data_files": [
+ {
+ "split": split_name,
+ "path": [
+ parquet_file["url"].replace("refs%2Fconvert%2Fparquet", parquet_commit_hash)
+ for parquet_file in parquet_files_for_split
+ ],
+ }
+ for split_name, parquet_files_for_split in groupby(parquet_files_for_config, itemgetter("split"))
+ ],
+ "version": str(dataset_infos.get(config_name, DatasetInfo()).version or "0.0.0"),
+ }
+ for config_name, parquet_files_for_config in groupby(exported_parquet_files, itemgetter("config"))
+ }
+ if dataset_infos:
+ # Preserve order of configs and splits
+ metadata_configs = {
+ config_name: {
+ "data_files": [
+ data_file
+ for split_name in dataset_info.splits
+ for data_file in metadata_configs[config_name]["data_files"]
+ if data_file["split"] == split_name
+ ],
+ "version": metadata_configs[config_name]["version"],
+ }
+ for config_name, dataset_info in dataset_infos.items()
+ }
+ return cls(metadata_configs)
+
+ @classmethod
+ def from_dataset_card_data(cls, dataset_card_data: DatasetCardData) -> "MetadataConfigs":
+ if dataset_card_data.get(cls.FIELD_NAME):
+ metadata_configs = dataset_card_data[cls.FIELD_NAME]
+ if not isinstance(metadata_configs, list):
+ raise ValueError(f"Expected {cls.FIELD_NAME} to be a list, but got '{metadata_configs}'")
+ for metadata_config in metadata_configs:
+ if "config_name" not in metadata_config:
+ raise ValueError(
+ f"Each config must include `config_name` field with a string name of a config, "
+ f"but got {metadata_config}. "
+ )
+ cls._raise_if_data_files_field_not_valid(metadata_config)
+ return cls(
+ {
+ config.pop("config_name"): {
+ param: value if param != "features" else Features._from_yaml_list(value)
+ for param, value in config.items()
+ }
+ for metadata_config in metadata_configs
+ if (config := metadata_config.copy())
+ }
+ )
+ return cls()
+
+ def to_dataset_card_data(self, dataset_card_data: DatasetCardData) -> None:
+ if self:
+ for metadata_config in self.values():
+ self._raise_if_data_files_field_not_valid(metadata_config)
+ current_metadata_configs = self.from_dataset_card_data(dataset_card_data)
+ total_metadata_configs = dict(sorted({**current_metadata_configs, **self}.items()))
+ for config_name, config_metadata in total_metadata_configs.items():
+ config_metadata.pop("config_name", None)
+ dataset_card_data[self.FIELD_NAME] = [
+ {"config_name": config_name, **config_metadata}
+ for config_name, config_metadata in total_metadata_configs.items()
+ ]
+
+ def get_default_config_name(self) -> Optional[str]:
+ default_config_name = None
+ for config_name, metadata_config in self.items():
+ if len(self) == 1 or config_name == "default" or metadata_config.get("default"):
+ if default_config_name is None:
+ default_config_name = config_name
+ else:
+ raise ValueError(
+ f"Dataset has several default configs: '{default_config_name}' and '{config_name}'."
+ )
+ return default_config_name
+
+
+# DEPRECATED - just here to support old versions of evaluate like 0.2.2
+# To support new tasks on the Hugging Face Hub, please open a PR for this file:
+# https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/src/pipelines.ts
+known_task_ids = {
+ "image-classification": [],
+ "translation": [],
+ "image-segmentation": [],
+ "fill-mask": [],
+ "automatic-speech-recognition": [],
+ "token-classification": [],
+ "sentence-similarity": [],
+ "audio-classification": [],
+ "question-answering": [],
+ "summarization": [],
+ "zero-shot-classification": [],
+ "table-to-text": [],
+ "feature-extraction": [],
+ "other": [],
+ "multiple-choice": [],
+ "text-classification": [],
+ "text-to-image": [],
+ "text2text-generation": [],
+ "zero-shot-image-classification": [],
+ "tabular-classification": [],
+ "tabular-regression": [],
+ "image-to-image": [],
+ "tabular-to-text": [],
+ "unconditional-image-generation": [],
+ "text-retrieval": [],
+ "text-to-speech": [],
+ "object-detection": [],
+ "audio-to-audio": [],
+ "text-generation": [],
+ "conversational": [],
+ "table-question-answering": [],
+ "visual-question-answering": [],
+ "image-to-text": [],
+ "reinforcement-learning": [],
+ "voice-activity-detection": [],
+ "time-series-forecasting": [],
+ "document-question-answering": [],
+}
diff --git a/venv/Lib/site-packages/datasets/utils/patching.py b/venv/Lib/site-packages/datasets/utils/patching.py
new file mode 100644
index 0000000000000000000000000000000000000000..69563f562e40b33fb9565ba9b6976279c9268b38
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/patching.py
@@ -0,0 +1,119 @@
+from importlib import import_module
+
+from .logging import get_logger
+
+
+logger = get_logger(__name__)
+
+
+class _PatchedModuleObj:
+ """Set all the modules components as attributes of the _PatchedModuleObj object."""
+
+ def __init__(self, module, attrs=None):
+ attrs = attrs or []
+ if module is not None:
+ for key in module.__dict__:
+ if key in attrs or not key.startswith("__"):
+ setattr(self, key, getattr(module, key))
+ self._original_module = module._original_module if isinstance(module, _PatchedModuleObj) else module
+
+
+class patch_submodule:
+ """
+ Patch a submodule attribute of an object, by keeping all other submodules intact at all levels.
+
+ Example::
+
+ >>> import importlib
+ >>> from datasets.load import dataset_module_factory
+ >>> from datasets.streaming import patch_submodule, xjoin
+ >>>
+ >>> dataset_module = dataset_module_factory("stanfordnlp/snli")
+ >>> snli_module = importlib.import_module(dataset_module.module_path)
+ >>> patcher = patch_submodule(snli_module, "os.path.join", xjoin)
+ >>> patcher.start()
+ >>> assert snli_module.os.path.join is xjoin
+ """
+
+ _active_patches = []
+
+ def __init__(self, obj, target: str, new, attrs=None):
+ self.obj = obj
+ self.target = target
+ self.new = new
+ self.key = target.split(".")[0]
+ self.original = {}
+ self.attrs = attrs or []
+
+ def __enter__(self):
+ *submodules, target_attr = self.target.split(".")
+
+ # Patch modules:
+ # it's used to patch attributes of submodules like "os.path.join";
+ # in this case we need to patch "os" and "os.path"
+
+ for i in range(len(submodules)):
+ try:
+ submodule = import_module(".".join(submodules[: i + 1]))
+ except ModuleNotFoundError:
+ continue
+ # We iterate over all the globals in self.obj in case we find "os" or "os.path"
+ for attr in self.obj.__dir__():
+ obj_attr = getattr(self.obj, attr)
+ # We don't check for the name of the global, but rather if its value *is* "os" or "os.path".
+ # This allows to patch renamed modules like "from os import path as ospath".
+ if obj_attr is submodule or (
+ isinstance(obj_attr, _PatchedModuleObj) and obj_attr._original_module is submodule
+ ):
+ self.original[attr] = obj_attr
+ # patch at top level
+ setattr(self.obj, attr, _PatchedModuleObj(obj_attr, attrs=self.attrs))
+ patched = getattr(self.obj, attr)
+ # construct lower levels patches
+ for key in submodules[i + 1 :]:
+ setattr(patched, key, _PatchedModuleObj(getattr(patched, key, None), attrs=self.attrs))
+ patched = getattr(patched, key)
+ # finally set the target attribute
+ setattr(patched, target_attr, self.new)
+
+ # Patch attribute itself:
+ # it's used for builtins like "open",
+ # and also to patch "os.path.join" we may also need to patch "join"
+ # itself if it was imported as "from os.path import join".
+
+ if submodules: # if it's an attribute of a submodule like "os.path.join"
+ try:
+ attr_value = getattr(import_module(".".join(submodules)), target_attr)
+ except (AttributeError, ModuleNotFoundError):
+ return
+ # We iterate over all the globals in self.obj in case we find "os.path.join"
+ for attr in self.obj.__dir__():
+ # We don't check for the name of the global, but rather if its value *is* "os.path.join".
+ # This allows to patch renamed attributes like "from os.path import join as pjoin".
+ if getattr(self.obj, attr) is attr_value:
+ self.original[attr] = getattr(self.obj, attr)
+ setattr(self.obj, attr, self.new)
+ elif target_attr in globals()["__builtins__"]: # if it'a s builtin like "open"
+ self.original[target_attr] = globals()["__builtins__"][target_attr]
+ setattr(self.obj, target_attr, self.new)
+ else:
+ raise RuntimeError(f"Tried to patch attribute {target_attr} instead of a submodule.")
+
+ def __exit__(self, *exc_info):
+ for attr in list(self.original):
+ setattr(self.obj, attr, self.original.pop(attr))
+
+ def start(self):
+ """Activate a patch."""
+ self.__enter__()
+ self._active_patches.append(self)
+
+ def stop(self):
+ """Stop an active patch."""
+ try:
+ self._active_patches.remove(self)
+ except ValueError:
+ # If the patch hasn't been started this will fail
+ return None
+
+ return self.__exit__()
diff --git a/venv/Lib/site-packages/datasets/utils/py_utils.py b/venv/Lib/site-packages/datasets/utils/py_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..df6d7b977b554cf772eeb32dcdf8dc5f0b0f2f4c
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/py_utils.py
@@ -0,0 +1,640 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Some python utils function and classes."""
+
+import copy
+import functools
+import itertools
+import multiprocessing.pool
+import os
+import queue
+import re
+import types
+import warnings
+from collections.abc import Iterable
+from contextlib import contextmanager
+from dataclasses import fields, is_dataclass
+from queue import Empty
+from shutil import disk_usage
+from typing import Any, Callable, Optional, TypeVar, Union
+
+import multiprocess
+import multiprocess.pool
+import numpy as np
+from tqdm.auto import tqdm
+
+from .. import config
+from ..parallel import parallel_map
+from . import logging
+from . import tqdm as hf_tqdm
+from ._dill import ( # noqa: F401 # imported for backward compatibility. TODO: remove in 3.0.0
+ Pickler,
+ dump,
+ dumps,
+ pklregister,
+)
+
+
+try: # pragma: no branch
+ from typing import Final
+
+ import typing_extensions as _typing_extensions
+ from typing_extensions import Literal
+except ImportError:
+ _typing_extensions = Literal = Final = None
+
+
+logger = logging.get_logger(__name__)
+
+
+# NOTE: When used on an instance method, the cache is shared across all
+# instances and IS NOT per-instance.
+# See
+# https://stackoverflow.com/questions/14946264/python-lru-cache-decorator-per-instance
+# For @property methods, use @memoized_property below.
+memoize = functools.lru_cache
+
+
+def size_str(size_in_bytes):
+ """Returns a human readable size string.
+
+ If size_in_bytes is None, then returns "Unknown size".
+
+ For example `size_str(1.5 * datasets.units.GiB) == "1.50 GiB"`.
+
+ Args:
+ size_in_bytes: `int` or `None`, the size, in bytes, that we want to
+ format as a human-readable size string.
+ """
+ if not size_in_bytes:
+ return "Unknown size"
+
+ _NAME_LIST = [("PiB", 2**50), ("TiB", 2**40), ("GiB", 2**30), ("MiB", 2**20), ("KiB", 2**10)]
+
+ size_in_bytes = float(size_in_bytes)
+ for name, size_bytes in _NAME_LIST:
+ value = size_in_bytes / size_bytes
+ if value >= 1.0:
+ return f"{value:.2f} {name}"
+ return f"{int(size_in_bytes)} bytes"
+
+
+def convert_file_size_to_int(size: Union[int, str]) -> int:
+ """
+ Converts a size expressed as a string with digits an unit (like `"50MB"`) to an integer (in bytes).
+
+ Args:
+ size (`int` or `str`): The size to convert. Will be directly returned if an `int`.
+
+ Example:
+
+ ```py
+ >>> convert_file_size_to_int("1MiB")
+ 1048576
+ ```
+ """
+ if isinstance(size, int):
+ return size
+ if size.upper().endswith("PIB"):
+ return int(size[:-3]) * (2**50)
+ if size.upper().endswith("TIB"):
+ return int(size[:-3]) * (2**40)
+ if size.upper().endswith("GIB"):
+ return int(size[:-3]) * (2**30)
+ if size.upper().endswith("MIB"):
+ return int(size[:-3]) * (2**20)
+ if size.upper().endswith("KIB"):
+ return int(size[:-3]) * (2**10)
+ if size.upper().endswith("PB"):
+ int_size = int(size[:-2]) * (10**15)
+ return int_size // 8 if size.endswith("b") else int_size
+ if size.upper().endswith("TB"):
+ int_size = int(size[:-2]) * (10**12)
+ return int_size // 8 if size.endswith("b") else int_size
+ if size.upper().endswith("GB"):
+ int_size = int(size[:-2]) * (10**9)
+ return int_size // 8 if size.endswith("b") else int_size
+ if size.upper().endswith("MB"):
+ int_size = int(size[:-2]) * (10**6)
+ return int_size // 8 if size.endswith("b") else int_size
+ if size.upper().endswith("KB"):
+ int_size = int(size[:-2]) * (10**3)
+ return int_size // 8 if size.endswith("b") else int_size
+ raise ValueError(f"`size={size}` is not in a valid format. Use an integer followed by the unit, e.g., '5GB'.")
+
+
+def glob_pattern_to_regex(pattern):
+ # partially taken from fsspec:
+ # https://github.com/fsspec/filesystem_spec/blob/697d0f8133d8a5fbc3926e4761d7ecd51337ce50/fsspec/asyn.py#L735
+ return (
+ pattern.replace("\\", r"\\")
+ .replace(".", r"\.")
+ .replace("*", ".*")
+ .replace("+", r"\+")
+ .replace("//", "/")
+ .replace("(", r"\(")
+ .replace(")", r"\)")
+ .replace("|", r"\|")
+ .replace("^", r"\^")
+ .replace("$", r"\$")
+ .rstrip("/")
+ .replace("?", ".")
+ )
+
+
+def string_to_dict(string: str, pattern: str) -> Optional[dict[str, str]]:
+ """Un-format a string using a python f-string pattern.
+ From https://stackoverflow.com/a/36838374
+
+ Example::
+
+ >>> p = 'hello, my name is {name} and I am a {age} year old {what}'
+ >>> s = p.format(name='cody', age=18, what='quarterback')
+ >>> s
+ 'hello, my name is cody and I am a 18 year old quarterback'
+ >>> string_to_dict(s, p)
+ {'age': '18', 'name': 'cody', 'what': 'quarterback'}
+
+ Args:
+ string (str): input string
+ pattern (str): pattern formatted like a python f-string
+ This can be a regex - so in case of un-formatting paths you should use posix paths.
+ Otherwise backslashes for windows paths can cause issues.
+
+ Returns:
+ Optional[dict[str, str]]: dictionary of variable -> value, retrieved from the input using the pattern, or
+ `None` if the string does not match the pattern.
+ """
+ pattern = re.sub(r"{([^:}]+)(?::[^}]+)?}", r"{\1}", pattern) # remove format specifiers, e.g. {rank:05d} -> {rank}
+ regex = re.sub(r"{(.+?)}", r"(?P<_\1>.+)", pattern)
+ result = re.search(regex, string)
+ if result is None:
+ return None
+ values = list(result.groups())
+ keys = re.findall(r"{(.+?)}", pattern)
+ _dict = dict(zip(keys, values))
+ return _dict
+
+
+def asdict(obj):
+ """Convert an object to its dictionary representation recursively.
+
+
+ """
+
+ # Implementation based on https://docs.python.org/3/library/dataclasses.html#dataclasses.asdict
+
+ def _is_dataclass_instance(obj):
+ # https://docs.python.org/3/library/dataclasses.html#dataclasses.is_dataclass
+ return is_dataclass(obj) and not isinstance(obj, type)
+
+ def _asdict_inner(obj):
+ if _is_dataclass_instance(obj):
+ result = {}
+ for f in fields(obj):
+ value = _asdict_inner(getattr(obj, f.name))
+ if not f.init or value != f.default or f.metadata.get("include_in_asdict_even_if_is_default", False):
+ result[f.name] = value
+ return result
+ elif isinstance(obj, tuple) and hasattr(obj, "_fields"):
+ # obj is a namedtuple
+ return type(obj)(*[_asdict_inner(v) for v in obj])
+ elif isinstance(obj, (list, tuple)):
+ # Assume we can create an object of this type by passing in a
+ # generator (which is not true for namedtuples, handled
+ # above).
+ return type(obj)(_asdict_inner(v) for v in obj)
+ elif isinstance(obj, dict):
+ return {_asdict_inner(k): _asdict_inner(v) for k, v in obj.items()}
+ else:
+ return copy.deepcopy(obj)
+
+ if not isinstance(obj, dict) and not _is_dataclass_instance(obj):
+ raise TypeError(f"{obj} is not a dict or a dataclass")
+
+ return _asdict_inner(obj)
+
+
+@contextmanager
+def temporary_assignment(obj, attr, value):
+ """Temporarily assign obj.attr to value."""
+ original = getattr(obj, attr, None)
+ setattr(obj, attr, value)
+ try:
+ yield
+ finally:
+ setattr(obj, attr, original)
+
+
+@contextmanager
+def temp_seed(seed: int, set_pytorch=False, set_tensorflow=False):
+ """Temporarily set the random seed. This works for python numpy, pytorch and tensorflow."""
+ np_state = np.random.get_state()
+ np.random.seed(seed)
+
+ if set_pytorch and config.TORCH_AVAILABLE:
+ import torch
+
+ torch_state = torch.random.get_rng_state()
+ torch.random.manual_seed(seed)
+
+ if torch.cuda.is_available():
+ torch_cuda_states = torch.cuda.get_rng_state_all()
+ torch.cuda.manual_seed_all(seed)
+
+ if set_tensorflow and config.TF_AVAILABLE:
+ import tensorflow as tf
+ from tensorflow.python.eager import context as tfpycontext
+
+ tf_state = tf.random.get_global_generator()
+ temp_gen = tf.random.Generator.from_seed(seed)
+ tf.random.set_global_generator(temp_gen)
+
+ if not tf.executing_eagerly():
+ raise ValueError("Setting random seed for TensorFlow is only available in eager mode")
+
+ tf_context = tfpycontext.context() # eager mode context
+ tf_seed = tf_context._seed
+ tf_rng_initialized = hasattr(tf_context, "_rng")
+ if tf_rng_initialized:
+ tf_rng = tf_context._rng
+ tf_context._set_global_seed(seed)
+
+ try:
+ yield
+ finally:
+ np.random.set_state(np_state)
+
+ if set_pytorch and config.TORCH_AVAILABLE:
+ torch.random.set_rng_state(torch_state)
+ if torch.cuda.is_available():
+ torch.cuda.set_rng_state_all(torch_cuda_states)
+
+ if set_tensorflow and config.TF_AVAILABLE:
+ tf.random.set_global_generator(tf_state)
+
+ tf_context._seed = tf_seed
+ if tf_rng_initialized:
+ tf_context._rng = tf_rng
+ else:
+ delattr(tf_context, "_rng")
+
+
+def unique_values(values):
+ """Iterate over iterable and return only unique values in order."""
+ seen = set()
+ for value in values:
+ if value not in seen:
+ seen.add(value)
+ yield value
+
+
+def no_op_if_value_is_null(func):
+ """If the value is None, return None, else call `func`."""
+
+ def wrapper(value):
+ return func(value) if value is not None else None
+
+ return wrapper
+
+
+def first_non_null_value(iterable):
+ """Return the index and the value of the first non-null value in the iterable. If all values are None, return -1 as index."""
+ for i, value in enumerate(iterable):
+ if value is not None:
+ return i, value
+ return -1, None
+
+
+def first_non_null_non_empty_value(iterable):
+ """Return the index and the value of the first non-null non-empty value in the iterable. If all values are None or empty, return -1 as index."""
+ for i, value in enumerate(iterable):
+ if value is not None and not (isinstance(value, (dict, list)) and len(value) == 0):
+ return i, value
+ return -1, None
+
+
+def zip_dict(*dicts):
+ """Iterate over items of dictionaries grouped by their keys."""
+ for key in unique_values(itertools.chain(*dicts)): # set merge all keys
+ # Will raise KeyError if the dict don't have the same keys
+ yield key, tuple(d[key] for d in dicts)
+
+
+class NonMutableDict(dict):
+ """Dict where keys can only be added but not modified.
+
+ Will raise an error if the user try to overwrite one key. The error message
+ can be customized during construction. It will be formatted using {key} for
+ the overwritten key.
+ """
+
+ def __init__(self, *args, **kwargs):
+ self._error_msg = kwargs.pop(
+ "error_msg",
+ "Try to overwrite existing key: {key}",
+ )
+ if kwargs:
+ raise ValueError("NonMutableDict cannot be initialized with kwargs.")
+ super().__init__(*args, **kwargs)
+
+ def __setitem__(self, key, value):
+ if key in self:
+ raise ValueError(self._error_msg.format(key=key))
+ return super().__setitem__(key, value)
+
+ def update(self, other):
+ if any(k in self for k in other):
+ raise ValueError(self._error_msg.format(key=set(self) & set(other)))
+ return super().update(other)
+
+
+class classproperty(property): # pylint: disable=invalid-name
+ """Descriptor to be used as decorator for @classmethods."""
+
+ def __get__(self, obj, objtype=None):
+ return self.fget.__get__(None, objtype)()
+
+
+def _single_map_nested(args):
+ """Apply a function recursively to each element of a nested data struct."""
+ function, data_struct, batched, batch_size, types, rank, disable_tqdm, desc = args
+
+ # Singleton first to spare some computation
+ if not isinstance(data_struct, dict) and not isinstance(data_struct, types):
+ if batched:
+ return function([data_struct])[0]
+ else:
+ return function(data_struct)
+ if (
+ batched
+ and not isinstance(data_struct, dict)
+ and isinstance(data_struct, types)
+ and all(not isinstance(v, (dict, types)) for v in data_struct)
+ ):
+ return [mapped_item for batch in iter_batched(data_struct, batch_size) for mapped_item in function(batch)]
+
+ # Reduce logging to keep things readable in multiprocessing with tqdm
+ if rank is not None and logging.get_verbosity() < logging.WARNING:
+ logging.set_verbosity_warning()
+ # Print at least one thing to fix tqdm in notebooks in multiprocessing
+ # see https://github.com/tqdm/tqdm/issues/485#issuecomment-473338308
+ if rank is not None and not disable_tqdm and any("notebook" in tqdm_cls.__name__ for tqdm_cls in tqdm.__mro__):
+ print(" ", end="", flush=True)
+
+ # Loop over single examples or batches and write to buffer/file if examples are to be updated
+ pbar_iterable = data_struct.items() if isinstance(data_struct, dict) else data_struct
+ pbar_desc = (desc + " " if desc is not None else "") + "#" + str(rank) if rank is not None else desc
+ with hf_tqdm(pbar_iterable, disable=disable_tqdm, position=rank, unit="obj", desc=pbar_desc) as pbar:
+ if isinstance(data_struct, dict):
+ return {
+ k: _single_map_nested((function, v, batched, batch_size, types, None, True, None)) for k, v in pbar
+ }
+ else:
+ mapped = [_single_map_nested((function, v, batched, batch_size, types, None, True, None)) for v in pbar]
+ if isinstance(data_struct, list):
+ return mapped
+ elif isinstance(data_struct, tuple):
+ return tuple(mapped)
+ else:
+ return np.array(mapped)
+
+
+def map_nested(
+ function: Callable[[Any], Any],
+ data_struct: Any,
+ dict_only: bool = False,
+ map_list: bool = True,
+ map_tuple: bool = False,
+ map_numpy: bool = False,
+ num_proc: Optional[int] = None,
+ parallel_min_length: int = 2,
+ batched: bool = False,
+ batch_size: Optional[int] = 1000,
+ types: Optional[tuple] = None,
+ disable_tqdm: bool = True,
+ desc: Optional[str] = None,
+) -> Any:
+ """Apply a function recursively to each element of a nested data struct.
+
+ Use multiprocessing if num_proc > 1 and the length of data_struct is greater than or equal to
+ `parallel_min_length`.
+
+
+
+ Before version 2.5.0, multiprocessing was not used if `num_proc` was greater than or equal to ``len(iterable)``.
+
+ Now, if `num_proc` is greater than or equal to ``len(iterable)``, `num_proc` is set to ``len(iterable)`` and
+ multiprocessing is used.
+
+
+
+ Args:
+ function (`Callable`): Function to be applied to `data_struct`.
+ data_struct (`Any`): Data structure to apply `function` to.
+ dict_only (`bool`, default `False`): Whether only apply `function` recursively to `dict` values in
+ `data_struct`.
+ map_list (`bool`, default `True`): Whether also apply `function` recursively to `list` elements (besides `dict`
+ values).
+ map_tuple (`bool`, default `False`): Whether also apply `function` recursively to `tuple` elements (besides
+ `dict` values).
+ map_numpy (`bool, default `False`): Whether also apply `function` recursively to `numpy.array` elements (besides
+ `dict` values).
+ num_proc (`int`, *optional*): Number of processes.
+ The level in the data struct used for multiprocessing is the first level that has smaller sub-structs,
+ starting from the root.
+ parallel_min_length (`int`, default `2`): Minimum length of `data_struct` required for parallel
+ processing.
+
+ batched (`bool`, defaults to `False`):
+ Provide batch of items to `function`.
+
+ batch_size (`int`, *optional*, defaults to `1000`):
+ Number of items per batch provided to `function` if `batched=True`.
+ If `batch_size <= 0` or `batch_size == None`, provide the full iterable as a single batch to `function`.
+
+ types (`tuple`, *optional*): Additional types (besides `dict` values) to apply `function` recursively to their
+ elements.
+ disable_tqdm (`bool`, default `True`): Whether to disable the tqdm progressbar.
+ desc (`str`, *optional*): Prefix for the tqdm progressbar.
+
+ Returns:
+ `Any`
+ """
+ if types is None:
+ types = []
+ if not dict_only:
+ if map_list:
+ types.append(list)
+ if map_tuple:
+ types.append(tuple)
+ if map_numpy:
+ types.append(np.ndarray)
+ types = tuple(types)
+
+ # Singleton
+ if not isinstance(data_struct, dict) and not isinstance(data_struct, types):
+ if batched:
+ data_struct = [data_struct]
+ mapped = function(data_struct)
+ if batched:
+ mapped = mapped[0]
+ return mapped
+
+ iterable = list(data_struct.values()) if isinstance(data_struct, dict) else data_struct
+
+ if num_proc is None:
+ num_proc = 1
+ if any(isinstance(v, types) and len(v) > len(iterable) for v in iterable):
+ mapped = [
+ map_nested(
+ function=function,
+ data_struct=obj,
+ num_proc=num_proc,
+ parallel_min_length=parallel_min_length,
+ batched=batched,
+ batch_size=batch_size,
+ types=types,
+ )
+ for obj in iterable
+ ]
+ elif num_proc != -1 and num_proc <= 1 or len(iterable) < parallel_min_length:
+ if batched:
+ if batch_size is None or batch_size <= 0:
+ batch_size = max(len(iterable) // num_proc + int(len(iterable) % num_proc > 0), 1)
+ iterable = list(iter_batched(iterable, batch_size))
+ mapped = [
+ _single_map_nested((function, obj, batched, batch_size, types, None, True, None))
+ for obj in hf_tqdm(iterable, disable=disable_tqdm, desc=desc)
+ ]
+ if batched:
+ mapped = [mapped_item for mapped_batch in mapped for mapped_item in mapped_batch]
+ else:
+ with warnings.catch_warnings():
+ warnings.filterwarnings(
+ "ignore",
+ message=".* is experimental and might be subject to breaking changes in the future\\.$",
+ category=UserWarning,
+ )
+ if batched:
+ if batch_size is None or batch_size <= 0:
+ batch_size = len(iterable) // num_proc + int(len(iterable) % num_proc > 0)
+ iterable = list(iter_batched(iterable, batch_size))
+ mapped = parallel_map(
+ function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, _single_map_nested
+ )
+ if batched:
+ mapped = [mapped_item for mapped_batch in mapped for mapped_item in mapped_batch]
+
+ if isinstance(data_struct, dict):
+ return dict(zip(data_struct.keys(), mapped))
+ else:
+ if isinstance(data_struct, list):
+ return mapped
+ elif isinstance(data_struct, tuple):
+ return tuple(mapped)
+ else:
+ return np.array(mapped)
+
+
+class NestedDataStructure:
+ def __init__(self, data=None):
+ self.data = data if data is not None else []
+
+ def flatten(self, data=None):
+ data = data if data is not None else self.data
+ if isinstance(data, dict):
+ return self.flatten(list(data.values()))
+ elif isinstance(data, (list, tuple)):
+ return [flattened for item in data for flattened in self.flatten(item)]
+ else:
+ return [data]
+
+
+def has_sufficient_disk_space(needed_bytes, directory="."):
+ try:
+ free_bytes = disk_usage(os.path.abspath(directory)).free
+ except OSError:
+ return True
+ return needed_bytes < free_bytes
+
+
+def copyfunc(func):
+ result = types.FunctionType(func.__code__, func.__globals__, func.__name__, func.__defaults__, func.__closure__)
+ result.__kwdefaults__ = func.__kwdefaults__
+ return result
+
+
+Y = TypeVar("Y")
+
+
+def _write_generator_to_queue(queue: queue.Queue, func: Callable[..., Iterable[Y]], kwargs: dict) -> int:
+ for i, result in enumerate(func(**kwargs)):
+ queue.put(result)
+ return i
+
+
+def _get_pool_pid(pool: Union[multiprocessing.pool.Pool, multiprocess.pool.Pool]) -> set[int]:
+ return {f.pid for f in pool._pool}
+
+
+def iflatmap_unordered(
+ pool: Union[multiprocessing.pool.Pool, multiprocess.pool.Pool],
+ func: Callable[..., Iterable[Y]],
+ *,
+ kwargs_iterable: Iterable[dict],
+) -> Iterable[Y]:
+ initial_pool_pid = _get_pool_pid(pool)
+ pool_changed = False
+ with pool._ctx.Manager() as manager:
+ queue = manager.Queue()
+ async_results = [
+ pool.apply_async(_write_generator_to_queue, (queue, func, kwargs)) for kwargs in kwargs_iterable
+ ]
+ try:
+ while True:
+ try:
+ yield queue.get(timeout=0.05)
+ except Empty:
+ if all(async_result.ready() for async_result in async_results) and queue.empty():
+ break
+ if _get_pool_pid(pool) != initial_pool_pid:
+ pool_changed = True
+ # One of the subprocesses has died. We should not wait forever.
+ raise RuntimeError(
+ "One of the subprocesses has abruptly died during map operation."
+ "To debug the error, disable multiprocessing."
+ )
+ finally:
+ if not pool_changed:
+ # we get the result in case there's an error to raise
+ [async_result.get(timeout=0.05) for async_result in async_results]
+
+
+T = TypeVar("T")
+
+
+def iter_batched(iterable: Iterable[T], n: int) -> Iterable[list[T]]:
+ if n < 1:
+ raise ValueError(f"Invalid batch size {n}")
+ batch = []
+ for item in iterable:
+ batch.append(item)
+ if len(batch) == n:
+ yield batch
+ batch = []
+ if batch:
+ yield batch
diff --git a/venv/Lib/site-packages/datasets/utils/resources/__init__.py b/venv/Lib/site-packages/datasets/utils/resources/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/venv/Lib/site-packages/datasets/utils/resources/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/datasets/utils/resources/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7407f78f431c91b1b24f4a0dabd4055c0494e763
Binary files /dev/null and b/venv/Lib/site-packages/datasets/utils/resources/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/datasets/utils/resources/creators.json b/venv/Lib/site-packages/datasets/utils/resources/creators.json
new file mode 100644
index 0000000000000000000000000000000000000000..d9e15f0039cc27ed8abd9fdf394423a3fada2c95
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/resources/creators.json
@@ -0,0 +1,17 @@
+{
+ "language": [
+ "found",
+ "crowdsourced",
+ "expert-generated",
+ "machine-generated",
+ "other"
+ ],
+ "annotations": [
+ "found",
+ "crowdsourced",
+ "expert-generated",
+ "machine-generated",
+ "no-annotation",
+ "other"
+ ]
+}
diff --git a/venv/Lib/site-packages/datasets/utils/resources/languages.json b/venv/Lib/site-packages/datasets/utils/resources/languages.json
new file mode 100644
index 0000000000000000000000000000000000000000..ea7686f956b898af3faf97b86be89b71d88855d4
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/resources/languages.json
@@ -0,0 +1,8026 @@
+{
+ "code": "Programming language (C++, Java, Javascript, Python, etc.)",
+ "aa": "Afar",
+ "aaa": "Ghotuo",
+ "aab": "Alumu-Tesu",
+ "aac": "Ari",
+ "aad": "Amal",
+ "aae": "Arbëreshë Albanian",
+ "aaf": "Aranadan",
+ "aag": "Ambrak",
+ "aah": "Abu' Arapesh",
+ "aai": "Arifama-Miniafia",
+ "aak": "Ankave",
+ "aal": "Afade",
+ "aan": "Anambé",
+ "aao": "Algerian Saharan Arabic",
+ "aap": "Pará Arára",
+ "aaq": "Eastern Abnaki",
+ "aas": "Aasáx",
+ "aat": "Arvanitika Albanian",
+ "aau": "Abau",
+ "aav": "Austro-Asiatic languages",
+ "aaw": "Solong",
+ "aax": "Mandobo Atas",
+ "aaz": "Amarasi",
+ "ab": "Abkhazian",
+ "aba": "Abé",
+ "abb": "Bankon",
+ "abc": "Ambala Ayta",
+ "abd": "Manide",
+ "abe": "Western Abnaki",
+ "abf": "Abai Sungai",
+ "abg": "Abaga",
+ "abh": "Tajiki Arabic",
+ "abi": "Abidji",
+ "abj": "Aka-Bea",
+ "abl": "Lampung Nyo",
+ "abm": "Abanyom",
+ "abn": "Abua",
+ "abo": "Abon",
+ "abp": "Abellen Ayta",
+ "abq": "Abaza",
+ "abr": "Abron",
+ "abs": "Ambonese Malay",
+ "abt": "Ambulas",
+ "abu": "Abure",
+ "abv": "Baharna Arabic",
+ "abw": "Pal",
+ "abx": "Inabaknon",
+ "aby": "Aneme Wake",
+ "abz": "Abui",
+ "aca": "Achagua",
+ "acb": "Áncá",
+ "acd": "Gikyode",
+ "ace": "Achinese",
+ "acf": "Saint Lucian Creole French",
+ "ach": "Acoli",
+ "aci": "Aka-Cari",
+ "ack": "Aka-Kora",
+ "acl": "Akar-Bale",
+ "acm": "Mesopotamian Arabic",
+ "acn": "Achang",
+ "acp": "Eastern Acipa",
+ "acq": "Ta'izzi-Adeni Arabic",
+ "acr": "Achi",
+ "acs": "Acroá",
+ "act": "Achterhoeks",
+ "acu": "Achuar-Shiwiar",
+ "acv": "Achumawi",
+ "acw": "Hijazi Arabic",
+ "acx": "Omani Arabic",
+ "acy": "Cypriot Arabic",
+ "acz": "Acheron",
+ "ada": "Adangme",
+ "adb": "Atauran",
+ "add": "Lidzonka; Dzodinka",
+ "ade": "Adele",
+ "adf": "Dhofari Arabic",
+ "adg": "Andegerebinha",
+ "adh": "Adhola",
+ "adi": "Adi",
+ "adj": "Adioukrou",
+ "adl": "Galo",
+ "adn": "Adang",
+ "ado": "Abu",
+ "adq": "Adangbe",
+ "adr": "Adonara",
+ "ads": "Adamorobe Sign Language",
+ "adt": "Adnyamathanha",
+ "adu": "Aduge",
+ "adw": "Amundava",
+ "adx": "Amdo Tibetan",
+ "ady": "Adyghe; Adygei",
+ "adz": "Adzera",
+ "ae": "Avestan",
+ "aea": "Areba",
+ "aeb": "Tunisian Arabic",
+ "aec": "Saidi Arabic",
+ "aed": "Argentine Sign Language",
+ "aee": "Northeast Pashai; Northeast Pashayi",
+ "aek": "Haeke",
+ "ael": "Ambele",
+ "aem": "Arem",
+ "aen": "Armenian Sign Language",
+ "aeq": "Aer",
+ "aer": "Eastern Arrernte",
+ "aes": "Alsea",
+ "aeu": "Akeu",
+ "aew": "Ambakich",
+ "aey": "Amele",
+ "aez": "Aeka",
+ "af": "Afrikaans",
+ "afa": "Afro-Asiatic languages",
+ "afb": "Gulf Arabic",
+ "afd": "Andai",
+ "afe": "Putukwam",
+ "afg": "Afghan Sign Language",
+ "afh": "Afrihili",
+ "afi": "Akrukay; Chini",
+ "afk": "Nanubae",
+ "afn": "Defaka",
+ "afo": "Eloyi",
+ "afp": "Tapei",
+ "afs": "Afro-Seminole Creole",
+ "aft": "Afitti",
+ "afu": "Awutu",
+ "afz": "Obokuitai",
+ "aga": "Aguano",
+ "agb": "Legbo",
+ "agc": "Agatu",
+ "agd": "Agarabi",
+ "age": "Angal",
+ "agf": "Arguni",
+ "agg": "Angor",
+ "agh": "Ngelima",
+ "agi": "Agariya",
+ "agj": "Argobba",
+ "agk": "Isarog Agta",
+ "agl": "Fembe",
+ "agm": "Angaataha",
+ "agn": "Agutaynen",
+ "ago": "Tainae",
+ "agq": "Aghem",
+ "agr": "Aguaruna",
+ "ags": "Esimbi",
+ "agt": "Central Cagayan Agta",
+ "agu": "Aguacateco",
+ "agv": "Remontado Dumagat",
+ "agw": "Kahua",
+ "agx": "Aghul",
+ "agy": "Southern Alta",
+ "agz": "Mt. Iriga Agta",
+ "aha": "Ahanta",
+ "ahb": "Axamb",
+ "ahg": "Qimant",
+ "ahh": "Aghu",
+ "ahi": "Tiagbamrin Aizi",
+ "ahk": "Akha",
+ "ahl": "Igo",
+ "ahm": "Mobumrin Aizi",
+ "ahn": "Àhàn",
+ "aho": "Ahom",
+ "ahp": "Aproumu Aizi",
+ "ahr": "Ahirani",
+ "ahs": "Ashe",
+ "aht": "Ahtena",
+ "aia": "Arosi",
+ "aib": "Ainu (China)",
+ "aic": "Ainbai",
+ "aid": "Alngith",
+ "aie": "Amara",
+ "aif": "Agi",
+ "aig": "Antigua and Barbuda Creole English",
+ "aih": "Ai-Cham",
+ "aii": "Assyrian Neo-Aramaic",
+ "aij": "Lishanid Noshan",
+ "aik": "Ake",
+ "ail": "Aimele",
+ "aim": "Aimol",
+ "ain": "Ainu (Japan)",
+ "aio": "Aiton",
+ "aip": "Burumakok",
+ "aiq": "Aimaq",
+ "air": "Airoran",
+ "ait": "Arikem",
+ "aiw": "Aari",
+ "aix": "Aighon",
+ "aiy": "Ali",
+ "aja": "Aja (South Sudan)",
+ "ajg": "Aja (Benin)",
+ "aji": "Ajië",
+ "ajn": "Andajin",
+ "ajp": "South Levantine Arabic",
+ "ajs": "Algerian Jewish Sign Language",
+ "aju": "Judeo-Moroccan Arabic",
+ "ajw": "Ajawa",
+ "ajz": "Amri Karbi",
+ "ak": "Akan",
+ "akb": "Batak Angkola",
+ "akc": "Mpur",
+ "akd": "Ukpet-Ehom",
+ "ake": "Akawaio",
+ "akf": "Akpa",
+ "akg": "Anakalangu",
+ "akh": "Angal Heneng",
+ "aki": "Aiome",
+ "akj": "Aka-Jeru",
+ "akk": "Akkadian",
+ "akl": "Aklanon",
+ "akm": "Aka-Bo",
+ "ako": "Akurio",
+ "akp": "Siwu",
+ "akq": "Ak",
+ "akr": "Araki",
+ "aks": "Akaselem",
+ "akt": "Akolet",
+ "aku": "Akum",
+ "akv": "Akhvakh",
+ "akw": "Akwa",
+ "akx": "Aka-Kede",
+ "aky": "Aka-Kol",
+ "akz": "Alabama",
+ "ala": "Alago",
+ "alc": "Qawasqar",
+ "ald": "Alladian",
+ "ale": "Aleut",
+ "alf": "Alege",
+ "alg": "Algonquian languages",
+ "alh": "Alawa",
+ "ali": "Amaimon",
+ "alj": "Alangan",
+ "alk": "Alak",
+ "all": "Allar",
+ "alm": "Amblong",
+ "aln": "Gheg Albanian",
+ "alo": "Larike-Wakasihu",
+ "alp": "Alune",
+ "alq": "Algonquin",
+ "alr": "Alutor",
+ "als": "Tosk Albanian",
+ "alt": "Southern Altai",
+ "alu": "'Are'are",
+ "alv": "Atlantic-Congo languages",
+ "alw": "Alaba-K’abeena; Wanbasana",
+ "alx": "Amol",
+ "aly": "Alyawarr",
+ "alz": "Alur",
+ "am": "Amharic",
+ "ama": "Amanayé",
+ "amb": "Ambo",
+ "amc": "Amahuaca",
+ "ame": "Yanesha'",
+ "amf": "Hamer-Banna",
+ "amg": "Amurdak",
+ "ami": "Amis",
+ "amj": "Amdang",
+ "amk": "Ambai",
+ "aml": "War-Jaintia",
+ "amm": "Ama (Papua New Guinea)",
+ "amn": "Amanab",
+ "amo": "Amo",
+ "amp": "Alamblak",
+ "amq": "Amahai",
+ "amr": "Amarakaeri",
+ "ams": "Southern Amami-Oshima",
+ "amt": "Amto",
+ "amu": "Guerrero Amuzgo",
+ "amv": "Ambelau",
+ "amw": "Western Neo-Aramaic",
+ "amx": "Anmatyerre",
+ "amy": "Ami",
+ "amz": "Atampaya",
+ "an": "Aragonese",
+ "ana": "Andaqui",
+ "anb": "Andoa",
+ "anc": "Ngas",
+ "and": "Ansus",
+ "ane": "Xârâcùù",
+ "anf": "Animere",
+ "ang": "Old English (ca. 450-1100)",
+ "anh": "Nend",
+ "ani": "Andi",
+ "anj": "Anor",
+ "ank": "Goemai",
+ "anl": "Anu-Hkongso Chin",
+ "anm": "Anal",
+ "ann": "Obolo",
+ "ano": "Andoque",
+ "anp": "Angika",
+ "anq": "Jarawa (India)",
+ "anr": "Andh",
+ "ans": "Anserma",
+ "ant": "Antakarinya; Antikarinya",
+ "anu": "Anuak",
+ "anv": "Denya",
+ "anw": "Anaang",
+ "anx": "Andra-Hus",
+ "any": "Anyin",
+ "anz": "Anem",
+ "aoa": "Angolar",
+ "aob": "Abom",
+ "aoc": "Pemon",
+ "aod": "Andarum",
+ "aoe": "Angal Enen",
+ "aof": "Bragat",
+ "aog": "Angoram",
+ "aoi": "Anindilyakwa",
+ "aoj": "Mufian",
+ "aok": "Arhö",
+ "aol": "Alor",
+ "aom": "Ömie",
+ "aon": "Bumbita Arapesh",
+ "aor": "Aore",
+ "aos": "Taikat",
+ "aot": "Atong (India); A'tong",
+ "aou": "A'ou",
+ "aox": "Atorada",
+ "aoz": "Uab Meto",
+ "apa": "Apache languages",
+ "apb": "Sa'a",
+ "apc": "North Levantine Arabic",
+ "apd": "Sudanese Arabic",
+ "ape": "Bukiyip",
+ "apf": "Pahanan Agta",
+ "apg": "Ampanang",
+ "aph": "Athpariya",
+ "api": "Apiaká",
+ "apj": "Jicarilla Apache",
+ "apk": "Kiowa Apache",
+ "apl": "Lipan Apache",
+ "apm": "Mescalero-Chiricahua Apache",
+ "apn": "Apinayé",
+ "apo": "Ambul",
+ "app": "Apma",
+ "apq": "A-Pucikwar",
+ "apr": "Arop-Lokep",
+ "aps": "Arop-Sissano",
+ "apt": "Apatani",
+ "apu": "Apurinã",
+ "apv": "Alapmunte",
+ "apw": "Western Apache",
+ "apx": "Aputai",
+ "apy": "Apalaí",
+ "apz": "Safeyoka",
+ "aqa": "Alacalufan languages",
+ "aqc": "Archi",
+ "aqd": "Ampari Dogon",
+ "aqg": "Arigidi",
+ "aqk": "Aninka",
+ "aql": "Algic languages",
+ "aqm": "Atohwaim",
+ "aqn": "Northern Alta",
+ "aqp": "Atakapa",
+ "aqr": "Arhâ",
+ "aqt": "Angaité",
+ "aqz": "Akuntsu",
+ "ar": "Arabic",
+ "arb": "Standard Arabic",
+ "arc": "Official Aramaic (700-300 BCE); Imperial Aramaic (700-300 BCE)",
+ "ard": "Arabana",
+ "are": "Western Arrarnta",
+ "arh": "Arhuaco",
+ "ari": "Arikara",
+ "arj": "Arapaso",
+ "ark": "Arikapú",
+ "arl": "Arabela",
+ "arn": "Mapudungun; Mapuche",
+ "aro": "Araona",
+ "arp": "Arapaho",
+ "arq": "Algerian Arabic",
+ "arr": "Karo (Brazil)",
+ "ars": "Najdi Arabic",
+ "art": "Artificial languages",
+ "aru": "Aruá (Amazonas State); Arawá",
+ "arv": "Arbore",
+ "arw": "Arawak",
+ "arx": "Aruá (Rodonia State)",
+ "ary": "Moroccan Arabic",
+ "arz": "Egyptian Arabic",
+ "as": "Assamese",
+ "asa": "Asu (Tanzania)",
+ "asb": "Assiniboine",
+ "asc": "Casuarina Coast Asmat",
+ "ase": "American Sign Language",
+ "asf": "Auslan; Australian Sign Language",
+ "asg": "Cishingini",
+ "ash": "Abishira",
+ "asi": "Buruwai",
+ "asj": "Sari",
+ "ask": "Ashkun",
+ "asl": "Asilulu",
+ "asn": "Xingú Asuriní",
+ "aso": "Dano",
+ "asp": "Algerian Sign Language",
+ "asq": "Austrian Sign Language",
+ "asr": "Asuri",
+ "ass": "Ipulo",
+ "ast": "Asturian; Asturleonese; Bable; Leonese",
+ "asu": "Tocantins Asurini",
+ "asv": "Asoa",
+ "asw": "Australian Aborigines Sign Language",
+ "asx": "Muratayak",
+ "asy": "Yaosakor Asmat",
+ "asz": "As",
+ "ata": "Pele-Ata",
+ "atb": "Zaiwa",
+ "atc": "Atsahuaca",
+ "atd": "Ata Manobo",
+ "ate": "Atemble",
+ "atg": "Ivbie North-Okpela-Arhe",
+ "ath": "Athapascan languages",
+ "ati": "Attié",
+ "atj": "Atikamekw",
+ "atk": "Ati",
+ "atl": "Mt. Iraya Agta",
+ "atm": "Ata",
+ "atn": "Ashtiani",
+ "ato": "Atong (Cameroon)",
+ "atp": "Pudtol Atta",
+ "atq": "Aralle-Tabulahan",
+ "atr": "Waimiri-Atroari",
+ "ats": "Gros Ventre",
+ "att": "Pamplona Atta",
+ "atu": "Reel",
+ "atv": "Northern Altai",
+ "atw": "Atsugewi",
+ "atx": "Arutani",
+ "aty": "Aneityum",
+ "atz": "Arta",
+ "aua": "Asumboa",
+ "aub": "Alugu",
+ "auc": "Waorani",
+ "aud": "Anuta",
+ "auf": "Arauan languages",
+ "aug": "Aguna",
+ "auh": "Aushi",
+ "aui": "Anuki",
+ "auj": "Awjilah",
+ "auk": "Heyo",
+ "aul": "Aulua",
+ "aum": "Asu (Nigeria)",
+ "aun": "Molmo One",
+ "auo": "Auyokawa",
+ "aup": "Makayam",
+ "auq": "Anus; Korur",
+ "aur": "Aruek",
+ "aus": "Australian languages",
+ "aut": "Austral",
+ "auu": "Auye",
+ "auw": "Awyi",
+ "aux": "Aurá",
+ "auy": "Awiyaana",
+ "auz": "Uzbeki Arabic",
+ "av": "Avaric",
+ "avb": "Avau",
+ "avd": "Alviri-Vidari",
+ "avi": "Avikam",
+ "avk": "Kotava",
+ "avl": "Eastern Egyptian Bedawi Arabic",
+ "avm": "Angkamuthi",
+ "avn": "Avatime",
+ "avo": "Agavotaguerra",
+ "avs": "Aushiri",
+ "avt": "Au",
+ "avu": "Avokaya",
+ "avv": "Avá-Canoeiro",
+ "awa": "Awadhi",
+ "awb": "Awa (Papua New Guinea)",
+ "awc": "Cicipu",
+ "awd": "Arawakan languages",
+ "awe": "Awetí",
+ "awg": "Anguthimri",
+ "awh": "Awbono",
+ "awi": "Aekyom",
+ "awk": "Awabakal",
+ "awm": "Arawum",
+ "awn": "Awngi",
+ "awo": "Awak",
+ "awr": "Awera",
+ "aws": "South Awyu",
+ "awt": "Araweté",
+ "awu": "Central Awyu",
+ "awv": "Jair Awyu",
+ "aww": "Awun",
+ "awx": "Awara",
+ "awy": "Edera Awyu",
+ "axb": "Abipon",
+ "axe": "Ayerrerenge",
+ "axg": "Mato Grosso Arára",
+ "axk": "Yaka (Central African Republic)",
+ "axl": "Lower Southern Aranda",
+ "axm": "Middle Armenian",
+ "axx": "Xârâgurè",
+ "ay": "Aymara",
+ "aya": "Awar",
+ "ayb": "Ayizo Gbe",
+ "ayc": "Southern Aymara",
+ "ayd": "Ayabadhu",
+ "aye": "Ayere",
+ "ayg": "Ginyanga",
+ "ayh": "Hadrami Arabic",
+ "ayi": "Leyigha",
+ "ayk": "Akuku",
+ "ayl": "Libyan Arabic",
+ "ayn": "Sanaani Arabic",
+ "ayo": "Ayoreo",
+ "ayp": "North Mesopotamian Arabic",
+ "ayq": "Ayi (Papua New Guinea)",
+ "ayr": "Central Aymara",
+ "ays": "Sorsogon Ayta",
+ "ayt": "Magbukun Ayta",
+ "ayu": "Ayu",
+ "ayz": "Mai Brat",
+ "az": "Azerbaijani",
+ "aza": "Azha",
+ "azb": "South Azerbaijani",
+ "azc": "Uto-Aztecan languages",
+ "azd": "Eastern Durango Nahuatl",
+ "azg": "San Pedro Amuzgos Amuzgo",
+ "azj": "North Azerbaijani",
+ "azm": "Ipalapa Amuzgo",
+ "azn": "Western Durango Nahuatl",
+ "azo": "Awing",
+ "azt": "Faire Atta",
+ "azz": "Highland Puebla Nahuatl",
+ "ba": "Bashkir",
+ "baa": "Babatana",
+ "bab": "Bainouk-Gunyuño",
+ "bac": "Badui",
+ "bad": "Banda languages",
+ "bae": "Baré",
+ "baf": "Nubaca",
+ "bag": "Tuki",
+ "bah": "Bahamas Creole English",
+ "bai": "Bamileke languages",
+ "baj": "Barakai",
+ "bal": "Baluchi",
+ "ban": "Balinese",
+ "bao": "Waimaha",
+ "bap": "Bantawa",
+ "bar": "Bavarian",
+ "bas": "Basa (Cameroon)",
+ "bat": "Baltic languages",
+ "bau": "Bada (Nigeria)",
+ "bav": "Vengo",
+ "baw": "Bambili-Bambui",
+ "bax": "Bamun",
+ "bay": "Batuley",
+ "bba": "Baatonum",
+ "bbb": "Barai",
+ "bbc": "Batak Toba",
+ "bbd": "Bau",
+ "bbe": "Bangba",
+ "bbf": "Baibai",
+ "bbg": "Barama",
+ "bbh": "Bugan",
+ "bbi": "Barombi",
+ "bbj": "Ghomálá'",
+ "bbk": "Babanki",
+ "bbl": "Bats",
+ "bbm": "Babango",
+ "bbn": "Uneapa",
+ "bbo": "Northern Bobo Madaré; Konabéré",
+ "bbp": "West Central Banda",
+ "bbq": "Bamali",
+ "bbr": "Girawa",
+ "bbs": "Bakpinka",
+ "bbt": "Mburku",
+ "bbu": "Kulung (Nigeria)",
+ "bbv": "Karnai",
+ "bbw": "Baba",
+ "bbx": "Bubia",
+ "bby": "Befang",
+ "bca": "Central Bai",
+ "bcb": "Bainouk-Samik",
+ "bcc": "Southern Balochi",
+ "bcd": "North Babar",
+ "bce": "Bamenyam",
+ "bcf": "Bamu",
+ "bcg": "Baga Pokur",
+ "bch": "Bariai",
+ "bci": "Baoulé",
+ "bcj": "Bardi",
+ "bck": "Bunuba",
+ "bcl": "Central Bikol",
+ "bcm": "Bannoni",
+ "bcn": "Bali (Nigeria)",
+ "bco": "Kaluli",
+ "bcp": "Bali (Democratic Republic of Congo)",
+ "bcq": "Bench",
+ "bcr": "Babine",
+ "bcs": "Kohumono",
+ "bct": "Bendi",
+ "bcu": "Awad Bing",
+ "bcv": "Shoo-Minda-Nye",
+ "bcw": "Bana",
+ "bcy": "Bacama",
+ "bcz": "Bainouk-Gunyaamolo",
+ "bda": "Bayot",
+ "bdb": "Basap",
+ "bdc": "Emberá-Baudó",
+ "bdd": "Bunama",
+ "bde": "Bade",
+ "bdf": "Biage",
+ "bdg": "Bonggi",
+ "bdh": "Baka (South Sudan)",
+ "bdi": "Burun",
+ "bdj": "Bai (South Sudan); Bai",
+ "bdk": "Budukh",
+ "bdl": "Indonesian Bajau",
+ "bdm": "Buduma",
+ "bdn": "Baldemu",
+ "bdo": "Morom",
+ "bdp": "Bende",
+ "bdq": "Bahnar",
+ "bdr": "West Coast Bajau",
+ "bds": "Burunge",
+ "bdt": "Bokoto",
+ "bdu": "Oroko",
+ "bdv": "Bodo Parja",
+ "bdw": "Baham",
+ "bdx": "Budong-Budong",
+ "bdy": "Bandjalang",
+ "bdz": "Badeshi",
+ "be": "Belarusian",
+ "bea": "Beaver",
+ "beb": "Bebele",
+ "bec": "Iceve-Maci",
+ "bed": "Bedoanas",
+ "bee": "Byangsi",
+ "bef": "Benabena",
+ "beg": "Belait",
+ "beh": "Biali",
+ "bei": "Bekati'",
+ "bej": "Beja; Bedawiyet",
+ "bek": "Bebeli",
+ "bem": "Bemba (Zambia)",
+ "beo": "Beami",
+ "bep": "Besoa",
+ "beq": "Beembe",
+ "ber": "Berber languages",
+ "bes": "Besme",
+ "bet": "Guiberoua Béte",
+ "beu": "Blagar",
+ "bev": "Daloa Bété",
+ "bew": "Betawi",
+ "bex": "Jur Modo",
+ "bey": "Beli (Papua New Guinea)",
+ "bez": "Bena (Tanzania)",
+ "bfa": "Bari",
+ "bfb": "Pauri Bareli",
+ "bfc": "Panyi Bai; Northern Bai",
+ "bfd": "Bafut",
+ "bfe": "Betaf; Tena",
+ "bff": "Bofi",
+ "bfg": "Busang Kayan",
+ "bfh": "Blafe",
+ "bfi": "British Sign Language",
+ "bfj": "Bafanji",
+ "bfk": "Ban Khor Sign Language",
+ "bfl": "Banda-Ndélé",
+ "bfm": "Mmen",
+ "bfn": "Bunak",
+ "bfo": "Malba Birifor",
+ "bfp": "Beba",
+ "bfq": "Badaga",
+ "bfr": "Bazigar",
+ "bfs": "Southern Bai",
+ "bft": "Balti",
+ "bfu": "Gahri",
+ "bfw": "Bondo",
+ "bfx": "Bantayanon",
+ "bfy": "Bagheli",
+ "bfz": "Mahasu Pahari",
+ "bg": "Bulgarian",
+ "bga": "Gwamhi-Wuri",
+ "bgb": "Bobongko",
+ "bgc": "Haryanvi",
+ "bgd": "Rathwi Bareli",
+ "bge": "Bauria",
+ "bgf": "Bangandu",
+ "bgg": "Bugun",
+ "bgi": "Giangan",
+ "bgj": "Bangolan",
+ "bgk": "Bit; Buxinhua",
+ "bgl": "Bo (Laos)",
+ "bgn": "Western Balochi",
+ "bgo": "Baga Koga",
+ "bgp": "Eastern Balochi",
+ "bgq": "Bagri",
+ "bgr": "Bawm Chin",
+ "bgs": "Tagabawa",
+ "bgt": "Bughotu",
+ "bgu": "Mbongno",
+ "bgv": "Warkay-Bipim",
+ "bgw": "Bhatri",
+ "bgx": "Balkan Gagauz Turkish",
+ "bgy": "Benggoi",
+ "bgz": "Banggai",
+ "bh": "Bihari languages",
+ "bha": "Bharia",
+ "bhb": "Bhili",
+ "bhc": "Biga",
+ "bhd": "Bhadrawahi",
+ "bhe": "Bhaya",
+ "bhf": "Odiai",
+ "bhg": "Binandere",
+ "bhh": "Bukharic",
+ "bhi": "Bhilali",
+ "bhj": "Bahing",
+ "bhl": "Bimin",
+ "bhm": "Bathari",
+ "bhn": "Bohtan Neo-Aramaic",
+ "bho": "Bhojpuri",
+ "bhp": "Bima",
+ "bhq": "Tukang Besi South",
+ "bhr": "Bara Malagasy",
+ "bhs": "Buwal",
+ "bht": "Bhattiyali",
+ "bhu": "Bhunjia",
+ "bhv": "Bahau",
+ "bhw": "Biak",
+ "bhx": "Bhalay",
+ "bhy": "Bhele",
+ "bhz": "Bada (Indonesia)",
+ "bi": "Bislama",
+ "bia": "Badimaya",
+ "bib": "Bissa; Bisa",
+ "bid": "Bidiyo",
+ "bie": "Bepour",
+ "bif": "Biafada",
+ "big": "Biangai",
+ "bik": "Bikol",
+ "bil": "Bile",
+ "bim": "Bimoba",
+ "bin": "Bini; Edo",
+ "bio": "Nai",
+ "bip": "Bila",
+ "biq": "Bipi",
+ "bir": "Bisorio",
+ "bit": "Berinomo",
+ "biu": "Biete",
+ "biv": "Southern Birifor",
+ "biw": "Kol (Cameroon)",
+ "bix": "Bijori",
+ "biy": "Birhor",
+ "biz": "Baloi",
+ "bja": "Budza",
+ "bjb": "Banggarla",
+ "bjc": "Bariji",
+ "bje": "Biao-Jiao Mien",
+ "bjf": "Barzani Jewish Neo-Aramaic",
+ "bjg": "Bidyogo",
+ "bjh": "Bahinemo",
+ "bji": "Burji",
+ "bjj": "Kanauji",
+ "bjk": "Barok",
+ "bjl": "Bulu (Papua New Guinea)",
+ "bjm": "Bajelani",
+ "bjn": "Banjar",
+ "bjo": "Mid-Southern Banda",
+ "bjp": "Fanamaket",
+ "bjr": "Binumarien",
+ "bjs": "Bajan",
+ "bjt": "Balanta-Ganja",
+ "bju": "Busuu",
+ "bjv": "Bedjond",
+ "bjw": "Bakwé",
+ "bjx": "Banao Itneg",
+ "bjy": "Bayali",
+ "bjz": "Baruga",
+ "bka": "Kyak",
+ "bkc": "Baka (Cameroon)",
+ "bkd": "Binukid; Talaandig",
+ "bkf": "Beeke",
+ "bkg": "Buraka",
+ "bkh": "Bakoko",
+ "bki": "Baki",
+ "bkj": "Pande",
+ "bkk": "Brokskat",
+ "bkl": "Berik",
+ "bkm": "Kom (Cameroon)",
+ "bkn": "Bukitan",
+ "bko": "Kwa'",
+ "bkp": "Boko (Democratic Republic of Congo)",
+ "bkq": "Bakairí",
+ "bkr": "Bakumpai",
+ "bks": "Northern Sorsoganon",
+ "bkt": "Boloki",
+ "bku": "Buhid",
+ "bkv": "Bekwarra",
+ "bkw": "Bekwel",
+ "bkx": "Baikeno",
+ "bky": "Bokyi",
+ "bkz": "Bungku",
+ "bla": "Siksika",
+ "blb": "Bilua",
+ "blc": "Bella Coola",
+ "bld": "Bolango",
+ "ble": "Balanta-Kentohe",
+ "blf": "Buol",
+ "blh": "Kuwaa",
+ "bli": "Bolia",
+ "blj": "Bolongan",
+ "blk": "Pa'o Karen; Pa'O",
+ "bll": "Biloxi",
+ "blm": "Beli (South Sudan)",
+ "bln": "Southern Catanduanes Bikol",
+ "blo": "Anii",
+ "blp": "Blablanga",
+ "blq": "Baluan-Pam",
+ "blr": "Blang",
+ "bls": "Balaesang",
+ "blt": "Tai Dam",
+ "blv": "Kibala; Bolo",
+ "blw": "Balangao",
+ "blx": "Mag-Indi Ayta",
+ "bly": "Notre",
+ "blz": "Balantak",
+ "bm": "Bambara",
+ "bma": "Lame",
+ "bmb": "Bembe",
+ "bmc": "Biem",
+ "bmd": "Baga Manduri",
+ "bme": "Limassa",
+ "bmf": "Bom-Kim",
+ "bmg": "Bamwe",
+ "bmh": "Kein",
+ "bmi": "Bagirmi",
+ "bmj": "Bote-Majhi",
+ "bmk": "Ghayavi",
+ "bml": "Bomboli",
+ "bmm": "Northern Betsimisaraka Malagasy",
+ "bmn": "Bina (Papua New Guinea)",
+ "bmo": "Bambalang",
+ "bmp": "Bulgebi",
+ "bmq": "Bomu",
+ "bmr": "Muinane",
+ "bms": "Bilma Kanuri",
+ "bmt": "Biao Mon",
+ "bmu": "Somba-Siawari",
+ "bmv": "Bum",
+ "bmw": "Bomwali",
+ "bmx": "Baimak",
+ "bmz": "Baramu",
+ "bn": "Bengali; Bangla",
+ "bna": "Bonerate",
+ "bnb": "Bookan",
+ "bnc": "Bontok",
+ "bnd": "Banda (Indonesia)",
+ "bne": "Bintauna",
+ "bnf": "Masiwang",
+ "bng": "Benga",
+ "bni": "Bangi",
+ "bnj": "Eastern Tawbuid",
+ "bnk": "Bierebo",
+ "bnl": "Boon",
+ "bnm": "Batanga",
+ "bnn": "Bunun",
+ "bno": "Bantoanon",
+ "bnp": "Bola",
+ "bnq": "Bantik",
+ "bnr": "Butmas-Tur",
+ "bns": "Bundeli",
+ "bnt": "Bantu languages",
+ "bnu": "Bentong",
+ "bnv": "Bonerif; Beneraf; Edwas",
+ "bnw": "Bisis",
+ "bnx": "Bangubangu",
+ "bny": "Bintulu",
+ "bnz": "Beezen",
+ "bo": "Tibetan",
+ "boa": "Bora",
+ "bob": "Aweer",
+ "boe": "Mundabli",
+ "bof": "Bolon",
+ "bog": "Bamako Sign Language",
+ "boh": "Boma",
+ "boi": "Barbareño",
+ "boj": "Anjam",
+ "bok": "Bonjo",
+ "bol": "Bole",
+ "bom": "Berom",
+ "bon": "Bine",
+ "boo": "Tiemacèwè Bozo",
+ "bop": "Bonkiman",
+ "boq": "Bogaya",
+ "bor": "Borôro",
+ "bot": "Bongo",
+ "bou": "Bondei",
+ "bov": "Tuwuli",
+ "bow": "Rema",
+ "box": "Buamu",
+ "boy": "Bodo (Central African Republic)",
+ "boz": "Tiéyaxo Bozo",
+ "bpa": "Daakaka",
+ "bpc": "Mbuk",
+ "bpd": "Banda-Banda",
+ "bpe": "Bauni",
+ "bpg": "Bonggo",
+ "bph": "Botlikh",
+ "bpi": "Bagupi",
+ "bpj": "Binji",
+ "bpk": "Orowe; 'Ôrôê",
+ "bpl": "Broome Pearling Lugger Pidgin",
+ "bpm": "Biyom",
+ "bpn": "Dzao Min",
+ "bpo": "Anasi",
+ "bpp": "Kaure",
+ "bpq": "Banda Malay",
+ "bpr": "Koronadal Blaan",
+ "bps": "Sarangani Blaan",
+ "bpt": "Barrow Point",
+ "bpu": "Bongu",
+ "bpv": "Bian Marind",
+ "bpw": "Bo (Papua New Guinea)",
+ "bpx": "Palya Bareli",
+ "bpy": "Bishnupriya",
+ "bpz": "Bilba",
+ "bqa": "Tchumbuli",
+ "bqb": "Bagusa",
+ "bqc": "Boko (Benin); Boo",
+ "bqd": "Bung",
+ "bqf": "Baga Kaloum",
+ "bqg": "Bago-Kusuntu",
+ "bqh": "Baima",
+ "bqi": "Bakhtiari",
+ "bqj": "Bandial",
+ "bqk": "Banda-Mbrès",
+ "bql": "Bilakura",
+ "bqm": "Wumboko",
+ "bqn": "Bulgarian Sign Language",
+ "bqo": "Balo",
+ "bqp": "Busa",
+ "bqq": "Biritai",
+ "bqr": "Burusu",
+ "bqs": "Bosngun",
+ "bqt": "Bamukumbit",
+ "bqu": "Boguru",
+ "bqv": "Koro Wachi; Begbere-Ejar",
+ "bqw": "Buru (Nigeria)",
+ "bqx": "Baangi",
+ "bqy": "Bengkala Sign Language",
+ "bqz": "Bakaka",
+ "br": "Breton",
+ "bra": "Braj",
+ "brb": "Brao; Lave",
+ "brc": "Berbice Creole Dutch",
+ "brd": "Baraamu",
+ "brf": "Bira",
+ "brg": "Baure",
+ "brh": "Brahui",
+ "bri": "Mokpwe",
+ "brj": "Bieria",
+ "brk": "Birked",
+ "brl": "Birwa",
+ "brm": "Barambu",
+ "brn": "Boruca",
+ "bro": "Brokkat",
+ "brp": "Barapasi",
+ "brq": "Breri",
+ "brr": "Birao",
+ "brs": "Baras",
+ "brt": "Bitare",
+ "bru": "Eastern Bru",
+ "brv": "Western Bru",
+ "brw": "Bellari",
+ "brx": "Bodo (India)",
+ "bry": "Burui",
+ "brz": "Bilbil",
+ "bs": "Bosnian",
+ "bsa": "Abinomn",
+ "bsb": "Brunei Bisaya",
+ "bsc": "Bassari; Oniyan",
+ "bse": "Wushi",
+ "bsf": "Bauchi",
+ "bsg": "Bashkardi",
+ "bsh": "Kati",
+ "bsi": "Bassossi",
+ "bsj": "Bangwinji",
+ "bsk": "Burushaski",
+ "bsl": "Basa-Gumna",
+ "bsm": "Busami",
+ "bsn": "Barasana-Eduria",
+ "bso": "Buso",
+ "bsp": "Baga Sitemu",
+ "bsq": "Bassa",
+ "bsr": "Bassa-Kontagora",
+ "bss": "Akoose",
+ "bst": "Basketo",
+ "bsu": "Bahonsuai",
+ "bsv": "Baga Sobané",
+ "bsw": "Baiso",
+ "bsx": "Yangkam",
+ "bsy": "Sabah Bisaya",
+ "bta": "Bata",
+ "btc": "Bati (Cameroon)",
+ "btd": "Batak Dairi",
+ "bte": "Gamo-Ningi",
+ "btf": "Birgit",
+ "btg": "Gagnoa Bété",
+ "bth": "Biatah Bidayuh",
+ "bti": "Burate",
+ "btj": "Bacanese Malay",
+ "btk": "Batak languages",
+ "btm": "Batak Mandailing",
+ "btn": "Ratagnon",
+ "bto": "Rinconada Bikol",
+ "btp": "Budibud",
+ "btq": "Batek",
+ "btr": "Baetora",
+ "bts": "Batak Simalungun",
+ "btt": "Bete-Bendi",
+ "btu": "Batu",
+ "btv": "Bateri",
+ "btw": "Butuanon",
+ "btx": "Batak Karo",
+ "bty": "Bobot",
+ "btz": "Batak Alas-Kluet",
+ "bua": "Buriat",
+ "bub": "Bua",
+ "buc": "Bushi",
+ "bud": "Ntcham",
+ "bue": "Beothuk",
+ "buf": "Bushoong",
+ "bug": "Buginese",
+ "buh": "Younuo Bunu",
+ "bui": "Bongili",
+ "buj": "Basa-Gurmana",
+ "buk": "Bugawac",
+ "bum": "Bulu (Cameroon)",
+ "bun": "Sherbro",
+ "buo": "Terei",
+ "bup": "Busoa",
+ "buq": "Brem",
+ "bus": "Bokobaru",
+ "but": "Bungain",
+ "buu": "Budu",
+ "buv": "Bun",
+ "buw": "Bubi",
+ "bux": "Boghom",
+ "buy": "Bullom So",
+ "buz": "Bukwen",
+ "bva": "Barein",
+ "bvb": "Bube",
+ "bvc": "Baelelea",
+ "bvd": "Baeggu",
+ "bve": "Berau Malay",
+ "bvf": "Boor",
+ "bvg": "Bonkeng",
+ "bvh": "Bure",
+ "bvi": "Belanda Viri",
+ "bvj": "Baan",
+ "bvk": "Bukat",
+ "bvl": "Bolivian Sign Language",
+ "bvm": "Bamunka",
+ "bvn": "Buna",
+ "bvo": "Bolgo",
+ "bvp": "Bumang",
+ "bvq": "Birri",
+ "bvr": "Burarra",
+ "bvt": "Bati (Indonesia)",
+ "bvu": "Bukit Malay",
+ "bvv": "Baniva",
+ "bvw": "Boga",
+ "bvx": "Dibole",
+ "bvy": "Baybayanon",
+ "bvz": "Bauzi",
+ "bwa": "Bwatoo",
+ "bwb": "Namosi-Naitasiri-Serua",
+ "bwc": "Bwile",
+ "bwd": "Bwaidoka",
+ "bwe": "Bwe Karen",
+ "bwf": "Boselewa",
+ "bwg": "Barwe",
+ "bwh": "Bishuo",
+ "bwi": "Baniwa",
+ "bwj": "Láá Láá Bwamu",
+ "bwk": "Bauwaki",
+ "bwl": "Bwela",
+ "bwm": "Biwat",
+ "bwn": "Wunai Bunu",
+ "bwo": "Boro (Ethiopia); Borna (Ethiopia)",
+ "bwp": "Mandobo Bawah",
+ "bwq": "Southern Bobo Madaré",
+ "bwr": "Bura-Pabir",
+ "bws": "Bomboma",
+ "bwt": "Bafaw-Balong",
+ "bwu": "Buli (Ghana)",
+ "bww": "Bwa",
+ "bwx": "Bu-Nao Bunu",
+ "bwy": "Cwi Bwamu",
+ "bwz": "Bwisi",
+ "bxa": "Tairaha",
+ "bxb": "Belanda Bor",
+ "bxc": "Molengue",
+ "bxd": "Pela",
+ "bxe": "Birale",
+ "bxf": "Bilur; Minigir",
+ "bxg": "Bangala",
+ "bxh": "Buhutu",
+ "bxi": "Pirlatapa",
+ "bxj": "Bayungu",
+ "bxk": "Bukusu; Lubukusu",
+ "bxl": "Jalkunan",
+ "bxm": "Mongolia Buriat",
+ "bxn": "Burduna",
+ "bxo": "Barikanchi",
+ "bxp": "Bebil",
+ "bxq": "Beele",
+ "bxr": "Russia Buriat",
+ "bxs": "Busam",
+ "bxu": "China Buriat",
+ "bxv": "Berakou",
+ "bxw": "Bankagooma",
+ "bxz": "Binahari",
+ "bya": "Batak",
+ "byb": "Bikya",
+ "byc": "Ubaghara",
+ "byd": "Benyadu'",
+ "bye": "Pouye",
+ "byf": "Bete",
+ "byg": "Baygo",
+ "byh": "Bhujel",
+ "byi": "Buyu",
+ "byj": "Bina (Nigeria)",
+ "byk": "Biao",
+ "byl": "Bayono",
+ "bym": "Bidjara",
+ "byn": "Bilin; Blin",
+ "byo": "Biyo",
+ "byp": "Bumaji",
+ "byq": "Basay",
+ "byr": "Baruya; Yipma",
+ "bys": "Burak",
+ "byt": "Berti",
+ "byv": "Medumba",
+ "byw": "Belhariya",
+ "byx": "Qaqet",
+ "byz": "Banaro",
+ "bza": "Bandi",
+ "bzb": "Andio",
+ "bzc": "Southern Betsimisaraka Malagasy",
+ "bzd": "Bribri",
+ "bze": "Jenaama Bozo",
+ "bzf": "Boikin",
+ "bzg": "Babuza",
+ "bzh": "Mapos Buang",
+ "bzi": "Bisu",
+ "bzj": "Belize Kriol English",
+ "bzk": "Nicaragua Creole English",
+ "bzl": "Boano (Sulawesi)",
+ "bzm": "Bolondo",
+ "bzn": "Boano (Maluku)",
+ "bzo": "Bozaba",
+ "bzp": "Kemberano",
+ "bzq": "Buli (Indonesia)",
+ "bzr": "Biri",
+ "bzs": "Brazilian Sign Language",
+ "bzt": "Brithenig",
+ "bzu": "Burmeso",
+ "bzv": "Naami",
+ "bzw": "Basa (Nigeria)",
+ "bzx": "Kɛlɛngaxo Bozo",
+ "bzy": "Obanliku",
+ "bzz": "Evant",
+ "ca": "Catalan; Valencian",
+ "caa": "Chortí",
+ "cab": "Garifuna",
+ "cac": "Chuj",
+ "cad": "Caddo",
+ "cae": "Lehar; Laalaa",
+ "caf": "Southern Carrier",
+ "cag": "Nivaclé",
+ "cah": "Cahuarano",
+ "cai": "Central American Indian languages",
+ "caj": "Chané",
+ "cak": "Kaqchikel; Cakchiquel",
+ "cal": "Carolinian",
+ "cam": "Cemuhî",
+ "can": "Chambri",
+ "cao": "Chácobo",
+ "cap": "Chipaya",
+ "caq": "Car Nicobarese",
+ "car": "Galibi Carib",
+ "cas": "Tsimané",
+ "cau": "Caucasian languages",
+ "cav": "Cavineña",
+ "caw": "Callawalla",
+ "cax": "Chiquitano",
+ "cay": "Cayuga",
+ "caz": "Canichana",
+ "cba": "Chibchan languages",
+ "cbb": "Cabiyarí",
+ "cbc": "Carapana",
+ "cbd": "Carijona",
+ "cbg": "Chimila",
+ "cbi": "Chachi",
+ "cbj": "Ede Cabe",
+ "cbk": "Chavacano",
+ "cbl": "Bualkhaw Chin",
+ "cbn": "Nyahkur",
+ "cbo": "Izora",
+ "cbq": "Tsucuba; Cuba",
+ "cbr": "Cashibo-Cacataibo",
+ "cbs": "Cashinahua",
+ "cbt": "Chayahuita",
+ "cbu": "Candoshi-Shapra",
+ "cbv": "Cacua",
+ "cbw": "Kinabalian",
+ "cby": "Carabayo",
+ "ccc": "Chamicuro",
+ "ccd": "Cafundo Creole",
+ "cce": "Chopi",
+ "ccg": "Samba Daka",
+ "cch": "Atsam",
+ "ccj": "Kasanga",
+ "ccl": "Cutchi-Swahili",
+ "ccm": "Malaccan Creole Malay",
+ "ccn": "North Caucasian languages",
+ "cco": "Comaltepec Chinantec",
+ "ccp": "Chakma",
+ "ccr": "Cacaopera",
+ "ccs": "South Caucasian languages",
+ "cda": "Choni",
+ "cdc": "Chadic languages",
+ "cdd": "Caddoan languages",
+ "cde": "Chenchu",
+ "cdf": "Chiru",
+ "cdh": "Chambeali",
+ "cdi": "Chodri",
+ "cdj": "Churahi",
+ "cdm": "Chepang",
+ "cdn": "Chaudangsi",
+ "cdo": "Min Dong Chinese",
+ "cdr": "Cinda-Regi-Tiyal",
+ "cds": "Chadian Sign Language",
+ "cdy": "Chadong",
+ "cdz": "Koda",
+ "ce": "Chechen",
+ "cea": "Lower Chehalis",
+ "ceb": "Cebuano",
+ "ceg": "Chamacoco",
+ "cek": "Eastern Khumi Chin",
+ "cel": "Celtic languages",
+ "cen": "Cen",
+ "cet": "Centúúm",
+ "cey": "Ekai Chin",
+ "cfa": "Dijim-Bwilim",
+ "cfd": "Cara",
+ "cfg": "Como Karim",
+ "cfm": "Falam Chin",
+ "cga": "Changriwa",
+ "cgc": "Kagayanen",
+ "cgg": "Chiga",
+ "cgk": "Chocangacakha",
+ "ch": "Chamorro",
+ "chb": "Chibcha",
+ "chc": "Catawba",
+ "chd": "Highland Oaxaca Chontal",
+ "chf": "Tabasco Chontal",
+ "chg": "Chagatai",
+ "chh": "Chinook",
+ "chj": "Ojitlán Chinantec",
+ "chk": "Chuukese",
+ "chl": "Cahuilla",
+ "chm": "Mari (Russia)",
+ "chn": "Chinook jargon",
+ "cho": "Choctaw",
+ "chp": "Chipewyan; Dene Suline",
+ "chq": "Quiotepec Chinantec",
+ "chr": "Cherokee",
+ "cht": "Cholón",
+ "chw": "Chuwabu",
+ "chx": "Chantyal",
+ "chy": "Cheyenne",
+ "chz": "Ozumacín Chinantec",
+ "cia": "Cia-Cia",
+ "cib": "Ci Gbe",
+ "cic": "Chickasaw",
+ "cid": "Chimariko",
+ "cie": "Cineni",
+ "cih": "Chinali",
+ "cik": "Chitkuli Kinnauri",
+ "cim": "Cimbrian",
+ "cin": "Cinta Larga",
+ "cip": "Chiapanec",
+ "cir": "Tiri; Haméa; Méa",
+ "ciw": "Chippewa",
+ "ciy": "Chaima",
+ "cja": "Western Cham",
+ "cje": "Chru",
+ "cjh": "Upper Chehalis",
+ "cji": "Chamalal",
+ "cjk": "Chokwe",
+ "cjm": "Eastern Cham",
+ "cjn": "Chenapian",
+ "cjo": "Ashéninka Pajonal",
+ "cjp": "Cabécar",
+ "cjs": "Shor",
+ "cjv": "Chuave",
+ "cjy": "Jinyu Chinese",
+ "ckb": "Central Kurdish",
+ "ckh": "Chak",
+ "ckl": "Cibak",
+ "ckm": "Chakavian",
+ "ckn": "Kaang Chin",
+ "cko": "Anufo",
+ "ckq": "Kajakse",
+ "ckr": "Kairak",
+ "cks": "Tayo",
+ "ckt": "Chukot",
+ "cku": "Koasati",
+ "ckv": "Kavalan",
+ "ckx": "Caka",
+ "cky": "Cakfem-Mushere",
+ "ckz": "Cakchiquel-Quiché Mixed Language",
+ "cla": "Ron",
+ "clc": "Chilcotin",
+ "cld": "Chaldean Neo-Aramaic",
+ "cle": "Lealao Chinantec",
+ "clh": "Chilisso",
+ "cli": "Chakali",
+ "clj": "Laitu Chin",
+ "clk": "Idu-Mishmi",
+ "cll": "Chala",
+ "clm": "Clallam",
+ "clo": "Lowland Oaxaca Chontal",
+ "clt": "Lautu Chin",
+ "clu": "Caluyanun",
+ "clw": "Chulym",
+ "cly": "Eastern Highland Chatino",
+ "cma": "Maa",
+ "cmc": "Chamic languages",
+ "cme": "Cerma",
+ "cmg": "Classical Mongolian",
+ "cmi": "Emberá-Chamí",
+ "cml": "Campalagian",
+ "cmm": "Michigamea",
+ "cmn": "Mandarin Chinese",
+ "cmo": "Central Mnong",
+ "cmr": "Mro-Khimi Chin",
+ "cms": "Messapic",
+ "cmt": "Camtho",
+ "cna": "Changthang",
+ "cnb": "Chinbon Chin",
+ "cnc": "Côông",
+ "cng": "Northern Qiang",
+ "cnh": "Hakha Chin; Haka Chin",
+ "cni": "Asháninka",
+ "cnk": "Khumi Chin",
+ "cnl": "Lalana Chinantec",
+ "cno": "Con",
+ "cnp": "Northern Ping Chinese; Northern Pinghua",
+ "cnq": "Chung",
+ "cnr": "Montenegrin",
+ "cns": "Central Asmat",
+ "cnt": "Tepetotutla Chinantec",
+ "cnu": "Chenoua",
+ "cnw": "Ngawn Chin",
+ "cnx": "Middle Cornish",
+ "co": "Corsican",
+ "coa": "Cocos Islands Malay",
+ "cob": "Chicomuceltec",
+ "coc": "Cocopa",
+ "cod": "Cocama-Cocamilla",
+ "coe": "Koreguaje",
+ "cof": "Colorado",
+ "cog": "Chong",
+ "coh": "Chonyi-Dzihana-Kauma; Chichonyi-Chidzihana-Chikauma",
+ "coj": "Cochimi",
+ "cok": "Santa Teresa Cora",
+ "col": "Columbia-Wenatchi",
+ "com": "Comanche",
+ "con": "Cofán",
+ "coo": "Comox",
+ "cop": "Coptic",
+ "coq": "Coquille",
+ "cot": "Caquinte",
+ "cou": "Wamey",
+ "cov": "Cao Miao",
+ "cow": "Cowlitz",
+ "cox": "Nanti",
+ "coz": "Chochotec",
+ "cpa": "Palantla Chinantec",
+ "cpb": "Ucayali-Yurúa Ashéninka",
+ "cpc": "Ajyíninka Apurucayali",
+ "cpe": "English-based creoles and pidgins",
+ "cpf": "French-based creoles and pidgins",
+ "cpg": "Cappadocian Greek",
+ "cpi": "Chinese Pidgin English",
+ "cpn": "Cherepon",
+ "cpo": "Kpeego",
+ "cpp": "Portuguese-based creoles and pidgins",
+ "cps": "Capiznon",
+ "cpu": "Pichis Ashéninka",
+ "cpx": "Pu-Xian Chinese",
+ "cpy": "South Ucayali Ashéninka",
+ "cqd": "Chuanqiandian Cluster Miao",
+ "cr": "Cree",
+ "cra": "Chara",
+ "crb": "Island Carib",
+ "crc": "Lonwolwol",
+ "crd": "Coeur d'Alene",
+ "crf": "Caramanta",
+ "crg": "Michif",
+ "crh": "Crimean Tatar; Crimean Turkish",
+ "cri": "Sãotomense",
+ "crj": "Southern East Cree",
+ "crk": "Plains Cree",
+ "crl": "Northern East Cree",
+ "crm": "Moose Cree",
+ "crn": "El Nayar Cora",
+ "cro": "Crow",
+ "crp": "Creoles and pidgins",
+ "crq": "Iyo'wujwa Chorote",
+ "crr": "Carolina Algonquian",
+ "crs": "Seselwa Creole French",
+ "crt": "Iyojwa'ja Chorote",
+ "crv": "Chaura",
+ "crw": "Chrau",
+ "crx": "Carrier",
+ "cry": "Cori",
+ "crz": "Cruzeño",
+ "cs": "Czech",
+ "csa": "Chiltepec Chinantec",
+ "csb": "Kashubian",
+ "csc": "Catalan Sign Language; Lengua de señas catalana; Llengua de Signes Catalana",
+ "csd": "Chiangmai Sign Language",
+ "cse": "Czech Sign Language",
+ "csf": "Cuba Sign Language",
+ "csg": "Chilean Sign Language",
+ "csh": "Asho Chin",
+ "csi": "Coast Miwok",
+ "csj": "Songlai Chin",
+ "csk": "Jola-Kasa",
+ "csl": "Chinese Sign Language",
+ "csm": "Central Sierra Miwok",
+ "csn": "Colombian Sign Language",
+ "cso": "Sochiapam Chinantec; Sochiapan Chinantec",
+ "csp": "Southern Ping Chinese; Southern Pinghua",
+ "csq": "Croatia Sign Language",
+ "csr": "Costa Rican Sign Language",
+ "css": "Southern Ohlone",
+ "cst": "Northern Ohlone",
+ "csu": "Central Sudanic languages",
+ "csv": "Sumtu Chin",
+ "csw": "Swampy Cree",
+ "csx": "Cambodian Sign Language",
+ "csy": "Siyin Chin",
+ "csz": "Coos",
+ "cta": "Tataltepec Chatino",
+ "ctc": "Chetco",
+ "ctd": "Tedim Chin",
+ "cte": "Tepinapa Chinantec",
+ "ctg": "Chittagonian",
+ "cth": "Thaiphum Chin",
+ "ctl": "Tlacoatzintepec Chinantec",
+ "ctm": "Chitimacha",
+ "ctn": "Chhintange",
+ "cto": "Emberá-Catío",
+ "ctp": "Western Highland Chatino",
+ "cts": "Northern Catanduanes Bikol",
+ "ctt": "Wayanad Chetti",
+ "ctu": "Chol",
+ "cty": "Moundadan Chetty",
+ "ctz": "Zacatepec Chatino",
+ "cu": "Church Slavic; Church Slavonic; Old Bulgarian; Old Church Slavonic; Old Slavonic",
+ "cua": "Cua",
+ "cub": "Cubeo",
+ "cuc": "Usila Chinantec",
+ "cuh": "Chuka; Gichuka",
+ "cui": "Cuiba",
+ "cuj": "Mashco Piro",
+ "cuk": "San Blas Kuna",
+ "cul": "Culina; Kulina",
+ "cuo": "Cumanagoto",
+ "cup": "Cupeño",
+ "cuq": "Cun",
+ "cur": "Chhulung",
+ "cus": "Cushitic languages",
+ "cut": "Teutila Cuicatec",
+ "cuu": "Tai Ya",
+ "cuv": "Cuvok",
+ "cuw": "Chukwa",
+ "cux": "Tepeuxila Cuicatec",
+ "cuy": "Cuitlatec",
+ "cv": "Chuvash",
+ "cvg": "Chug",
+ "cvn": "Valle Nacional Chinantec",
+ "cwa": "Kabwa",
+ "cwb": "Maindo",
+ "cwd": "Woods Cree",
+ "cwe": "Kwere",
+ "cwg": "Chewong; Cheq Wong",
+ "cwt": "Kuwaataay",
+ "cy": "Welsh",
+ "cya": "Nopala Chatino",
+ "cyb": "Cayubaba",
+ "cyo": "Cuyonon",
+ "czh": "Huizhou Chinese",
+ "czk": "Knaanic",
+ "czn": "Zenzontepec Chatino",
+ "czo": "Min Zhong Chinese",
+ "czt": "Zotung Chin",
+ "da": "Danish",
+ "daa": "Dangaléat",
+ "dac": "Dambi",
+ "dad": "Marik",
+ "dae": "Duupa",
+ "dag": "Dagbani",
+ "dah": "Gwahatike",
+ "dai": "Day",
+ "daj": "Dar Fur Daju",
+ "dak": "Dakota",
+ "dal": "Dahalo",
+ "dam": "Damakawa",
+ "dao": "Daai Chin",
+ "daq": "Dandami Maria",
+ "dar": "Dargwa",
+ "das": "Daho-Doo",
+ "dau": "Dar Sila Daju",
+ "dav": "Taita; Dawida",
+ "daw": "Davawenyo",
+ "dax": "Dayi",
+ "day": "Land Dayak languages",
+ "daz": "Dao",
+ "dba": "Bangime",
+ "dbb": "Deno",
+ "dbd": "Dadiya",
+ "dbe": "Dabe",
+ "dbf": "Edopi",
+ "dbg": "Dogul Dom Dogon",
+ "dbi": "Doka",
+ "dbj": "Ida'an",
+ "dbl": "Dyirbal",
+ "dbm": "Duguri",
+ "dbn": "Duriankere",
+ "dbo": "Dulbu",
+ "dbp": "Duwai",
+ "dbq": "Daba",
+ "dbr": "Dabarre",
+ "dbt": "Ben Tey Dogon",
+ "dbu": "Bondum Dom Dogon",
+ "dbv": "Dungu",
+ "dbw": "Bankan Tey Dogon",
+ "dby": "Dibiyaso",
+ "dcc": "Deccan",
+ "dcr": "Negerhollands",
+ "dda": "Dadi Dadi",
+ "ddd": "Dongotono",
+ "dde": "Doondo",
+ "ddg": "Fataluku",
+ "ddi": "West Goodenough",
+ "ddj": "Jaru",
+ "ddn": "Dendi (Benin)",
+ "ddo": "Dido",
+ "ddr": "Dhudhuroa",
+ "dds": "Donno So Dogon",
+ "ddw": "Dawera-Daweloor",
+ "de": "German",
+ "dec": "Dagik",
+ "ded": "Dedua",
+ "dee": "Dewoin",
+ "def": "Dezfuli",
+ "deg": "Degema",
+ "deh": "Dehwari",
+ "dei": "Demisa",
+ "dek": "Dek",
+ "del": "Delaware",
+ "dem": "Dem",
+ "den": "Slave (Athapascan)",
+ "dep": "Pidgin Delaware",
+ "deq": "Dendi (Central African Republic)",
+ "der": "Deori",
+ "des": "Desano",
+ "dev": "Domung",
+ "dez": "Dengese",
+ "dga": "Southern Dagaare",
+ "dgb": "Bunoge Dogon",
+ "dgc": "Casiguran Dumagat Agta",
+ "dgd": "Dagaari Dioula",
+ "dge": "Degenan",
+ "dgg": "Doga",
+ "dgh": "Dghwede",
+ "dgi": "Northern Dagara",
+ "dgk": "Dagba",
+ "dgl": "Andaandi; Dongolawi",
+ "dgn": "Dagoman",
+ "dgo": "Dogri (individual language)",
+ "dgr": "Dogrib; Tłı̨chǫ",
+ "dgs": "Dogoso",
+ "dgt": "Ndra'ngith",
+ "dgw": "Daungwurrung",
+ "dgx": "Doghoro",
+ "dgz": "Daga",
+ "dhd": "Dhundari",
+ "dhg": "Dhangu-Djangu; Dhangu; Djangu",
+ "dhi": "Dhimal",
+ "dhl": "Dhalandji",
+ "dhm": "Zemba",
+ "dhn": "Dhanki",
+ "dho": "Dhodia",
+ "dhr": "Dhargari",
+ "dhs": "Dhaiso",
+ "dhu": "Dhurga",
+ "dhv": "Dehu; Drehu",
+ "dhw": "Dhanwar (Nepal)",
+ "dhx": "Dhungaloo",
+ "dia": "Dia",
+ "dib": "South Central Dinka",
+ "dic": "Lakota Dida",
+ "did": "Didinga",
+ "dif": "Dieri; Diyari",
+ "dig": "Digo; Chidigo",
+ "dih": "Kumiai",
+ "dii": "Dimbong",
+ "dij": "Dai",
+ "dik": "Southwestern Dinka",
+ "dil": "Dilling",
+ "dim": "Dime",
+ "din": "Dinka",
+ "dio": "Dibo",
+ "dip": "Northeastern Dinka",
+ "diq": "Dimli (individual language)",
+ "dir": "Dirim",
+ "dis": "Dimasa",
+ "diu": "Diriku",
+ "diw": "Northwestern Dinka",
+ "dix": "Dixon Reef",
+ "diy": "Diuwe",
+ "diz": "Ding",
+ "dja": "Djadjawurrung",
+ "djb": "Djinba",
+ "djc": "Dar Daju Daju",
+ "djd": "Djamindjung; Ngaliwurru",
+ "dje": "Zarma",
+ "djf": "Djangun",
+ "dji": "Djinang",
+ "djj": "Djeebbana",
+ "djk": "Eastern Maroon Creole; Businenge Tongo; Nenge",
+ "djm": "Jamsay Dogon",
+ "djn": "Jawoyn; Djauan",
+ "djo": "Jangkang",
+ "djr": "Djambarrpuyngu",
+ "dju": "Kapriman",
+ "djw": "Djawi",
+ "dka": "Dakpakha",
+ "dkg": "Kadung",
+ "dkk": "Dakka",
+ "dkr": "Kuijau",
+ "dks": "Southeastern Dinka",
+ "dkx": "Mazagway",
+ "dlg": "Dolgan",
+ "dlk": "Dahalik",
+ "dlm": "Dalmatian",
+ "dln": "Darlong",
+ "dma": "Duma",
+ "dmb": "Mombo Dogon",
+ "dmc": "Gavak",
+ "dmd": "Madhi Madhi",
+ "dme": "Dugwor",
+ "dmf": "Medefaidrin",
+ "dmg": "Upper Kinabatangan",
+ "dmk": "Domaaki",
+ "dml": "Dameli",
+ "dmm": "Dama",
+ "dmn": "Mande languages",
+ "dmo": "Kemedzung",
+ "dmr": "East Damar",
+ "dms": "Dampelas",
+ "dmu": "Dubu; Tebi",
+ "dmv": "Dumpas",
+ "dmw": "Mudburra",
+ "dmx": "Dema",
+ "dmy": "Demta; Sowari",
+ "dna": "Upper Grand Valley Dani",
+ "dnd": "Daonda",
+ "dne": "Ndendeule",
+ "dng": "Dungan",
+ "dni": "Lower Grand Valley Dani",
+ "dnj": "Dan",
+ "dnk": "Dengka",
+ "dnn": "Dzùùngoo",
+ "dno": "Ndrulo; Northern Lendu",
+ "dnr": "Danaru",
+ "dnt": "Mid Grand Valley Dani",
+ "dnu": "Danau",
+ "dnv": "Danu",
+ "dnw": "Western Dani",
+ "dny": "Dení",
+ "doa": "Dom",
+ "dob": "Dobu",
+ "doc": "Northern Dong",
+ "doe": "Doe",
+ "dof": "Domu",
+ "doh": "Dong",
+ "doi": "Dogri (macrolanguage)",
+ "dok": "Dondo",
+ "dol": "Doso",
+ "don": "Toura (Papua New Guinea)",
+ "doo": "Dongo",
+ "dop": "Lukpa",
+ "doq": "Dominican Sign Language",
+ "dor": "Dori'o",
+ "dos": "Dogosé",
+ "dot": "Dass",
+ "dov": "Dombe",
+ "dow": "Doyayo",
+ "dox": "Bussa",
+ "doy": "Dompo",
+ "doz": "Dorze",
+ "dpp": "Papar",
+ "dra": "Dravidian languages",
+ "drb": "Dair",
+ "drc": "Minderico",
+ "drd": "Darmiya",
+ "dre": "Dolpo",
+ "drg": "Rungus",
+ "dri": "C'Lela",
+ "drl": "Paakantyi",
+ "drn": "West Damar",
+ "dro": "Daro-Matu Melanau",
+ "drq": "Dura",
+ "drs": "Gedeo",
+ "drt": "Drents",
+ "dru": "Rukai",
+ "dry": "Darai",
+ "dsb": "Lower Sorbian",
+ "dse": "Dutch Sign Language",
+ "dsh": "Daasanach",
+ "dsi": "Disa",
+ "dsl": "Danish Sign Language",
+ "dsn": "Dusner",
+ "dso": "Desiya",
+ "dsq": "Tadaksahak",
+ "dsz": "Mardin Sign Language",
+ "dta": "Daur",
+ "dtb": "Labuk-Kinabatangan Kadazan",
+ "dtd": "Ditidaht",
+ "dth": "Adithinngithigh",
+ "dti": "Ana Tinga Dogon",
+ "dtk": "Tene Kan Dogon",
+ "dtm": "Tomo Kan Dogon",
+ "dtn": "Daatsʼíin",
+ "dto": "Tommo So Dogon",
+ "dtp": "Kadazan Dusun; Central Dusun",
+ "dtr": "Lotud",
+ "dts": "Toro So Dogon",
+ "dtt": "Toro Tegu Dogon",
+ "dtu": "Tebul Ure Dogon",
+ "dty": "Dotyali",
+ "dua": "Duala",
+ "dub": "Dubli",
+ "duc": "Duna",
+ "due": "Umiray Dumaget Agta",
+ "duf": "Dumbea; Drubea",
+ "dug": "Duruma; Chiduruma",
+ "duh": "Dungra Bhil",
+ "dui": "Dumun",
+ "duk": "Uyajitaya",
+ "dul": "Alabat Island Agta",
+ "dum": "Middle Dutch (ca. 1050-1350)",
+ "dun": "Dusun Deyah",
+ "duo": "Dupaninan Agta",
+ "dup": "Duano",
+ "duq": "Dusun Malang",
+ "dur": "Dii",
+ "dus": "Dumi",
+ "duu": "Drung",
+ "duv": "Duvle",
+ "duw": "Dusun Witu",
+ "dux": "Duungooma",
+ "duy": "Dicamay Agta",
+ "duz": "Duli-Gey",
+ "dv": "Dhivehi; Divehi; Maldivian",
+ "dva": "Duau",
+ "dwa": "Diri",
+ "dwk": "Dawik Kui",
+ "dwr": "Dawro",
+ "dws": "Dutton World Speedwords",
+ "dwu": "Dhuwal",
+ "dww": "Dawawa",
+ "dwy": "Dhuwaya",
+ "dwz": "Dewas Rai",
+ "dya": "Dyan",
+ "dyb": "Dyaberdyaber",
+ "dyd": "Dyugun",
+ "dyg": "Villa Viciosa Agta",
+ "dyi": "Djimini Senoufo",
+ "dym": "Yanda Dom Dogon",
+ "dyn": "Dyangadi; Dhanggatti",
+ "dyo": "Jola-Fonyi",
+ "dyu": "Dyula",
+ "dyy": "Djabugay; Dyaabugay",
+ "dz": "Dzongkha",
+ "dza": "Tunzu",
+ "dze": "Djiwarli",
+ "dzg": "Dazaga",
+ "dzl": "Dzalakha",
+ "dzn": "Dzando",
+ "eaa": "Karenggapa",
+ "ebc": "Beginci",
+ "ebg": "Ebughu",
+ "ebk": "Eastern Bontok",
+ "ebo": "Teke-Ebo",
+ "ebr": "Ebrié",
+ "ebu": "Embu; Kiembu",
+ "ecr": "Eteocretan",
+ "ecs": "Ecuadorian Sign Language",
+ "ecy": "Eteocypriot",
+ "ee": "Ewe",
+ "eee": "E",
+ "efa": "Efai",
+ "efe": "Efe",
+ "efi": "Efik",
+ "ega": "Ega",
+ "egl": "Emilian",
+ "egm": "Benamanga",
+ "ego": "Eggon",
+ "egx": "Egyptian languages",
+ "egy": "Egyptian (Ancient)",
+ "ehs": "Miyakubo Sign Language",
+ "ehu": "Ehueun",
+ "eip": "Eipomek",
+ "eit": "Eitiep",
+ "eiv": "Askopan",
+ "eja": "Ejamat",
+ "eka": "Ekajuk",
+ "eke": "Ekit",
+ "ekg": "Ekari",
+ "eki": "Eki",
+ "ekk": "Standard Estonian",
+ "ekl": "Kol (Bangladesh); Kol",
+ "ekm": "Elip",
+ "eko": "Koti",
+ "ekp": "Ekpeye",
+ "ekr": "Yace",
+ "eky": "Eastern Kayah",
+ "el": "Modern Greek (1453-)",
+ "ele": "Elepi",
+ "elh": "El Hugeirat",
+ "eli": "Nding",
+ "elk": "Elkei",
+ "elm": "Eleme",
+ "elo": "El Molo",
+ "elu": "Elu",
+ "elx": "Elamite",
+ "ema": "Emai-Iuleha-Ora",
+ "emb": "Embaloh",
+ "eme": "Emerillon",
+ "emg": "Eastern Meohang",
+ "emi": "Mussau-Emira",
+ "emk": "Eastern Maninkakan",
+ "emm": "Mamulique",
+ "emn": "Eman",
+ "emp": "Northern Emberá",
+ "emq": "Eastern Minyag",
+ "ems": "Pacific Gulf Yupik",
+ "emu": "Eastern Muria",
+ "emw": "Emplawas",
+ "emx": "Erromintxela",
+ "emy": "Epigraphic Mayan",
+ "emz": "Mbessa",
+ "en": "English",
+ "ena": "Apali",
+ "enb": "Markweeta",
+ "enc": "En",
+ "end": "Ende",
+ "enf": "Forest Enets",
+ "enh": "Tundra Enets",
+ "enl": "Enlhet",
+ "enm": "Middle English (1100-1500)",
+ "enn": "Engenni",
+ "eno": "Enggano",
+ "enq": "Enga",
+ "enr": "Emumu; Emem",
+ "enu": "Enu",
+ "env": "Enwan (Edo State)",
+ "enw": "Enwan (Akwa Ibom State)",
+ "enx": "Enxet",
+ "eo": "Esperanto",
+ "eot": "Beti (Côte d'Ivoire)",
+ "epi": "Epie",
+ "era": "Eravallan",
+ "erg": "Sie",
+ "erh": "Eruwa",
+ "eri": "Ogea",
+ "erk": "South Efate",
+ "ero": "Horpa",
+ "err": "Erre",
+ "ers": "Ersu",
+ "ert": "Eritai",
+ "erw": "Erokwanas",
+ "es": "Spanish; Castilian",
+ "ese": "Ese Ejja",
+ "esg": "Aheri Gondi",
+ "esh": "Eshtehardi",
+ "esi": "North Alaskan Inupiatun",
+ "esk": "Northwest Alaska Inupiatun",
+ "esl": "Egypt Sign Language",
+ "esm": "Esuma",
+ "esn": "Salvadoran Sign Language",
+ "eso": "Estonian Sign Language",
+ "esq": "Esselen",
+ "ess": "Central Siberian Yupik",
+ "esu": "Central Yupik",
+ "esx": "Eskimo-Aleut languages",
+ "esy": "Eskayan",
+ "et": "Estonian",
+ "etb": "Etebi",
+ "etc": "Etchemin",
+ "eth": "Ethiopian Sign Language",
+ "etn": "Eton (Vanuatu)",
+ "eto": "Eton (Cameroon)",
+ "etr": "Edolo",
+ "ets": "Yekhee",
+ "ett": "Etruscan",
+ "etu": "Ejagham",
+ "etx": "Eten",
+ "etz": "Semimi",
+ "eu": "Basque",
+ "euq": "Basque (family)",
+ "eve": "Even",
+ "evh": "Uvbie",
+ "evn": "Evenki",
+ "ewo": "Ewondo",
+ "ext": "Extremaduran",
+ "eya": "Eyak",
+ "eyo": "Keiyo",
+ "eza": "Ezaa",
+ "eze": "Uzekwe",
+ "fa": "Persian",
+ "faa": "Fasu",
+ "fab": "Fa d'Ambu",
+ "fad": "Wagi",
+ "faf": "Fagani",
+ "fag": "Finongan",
+ "fah": "Baissa Fali",
+ "fai": "Faiwol",
+ "faj": "Faita",
+ "fak": "Fang (Cameroon)",
+ "fal": "South Fali",
+ "fam": "Fam",
+ "fan": "Fang (Equatorial Guinea)",
+ "fap": "Paloor",
+ "far": "Fataleka",
+ "fat": "Fanti",
+ "fau": "Fayu",
+ "fax": "Fala",
+ "fay": "Southwestern Fars",
+ "faz": "Northwestern Fars",
+ "fbl": "West Albay Bikol",
+ "fcs": "Quebec Sign Language",
+ "fer": "Feroge",
+ "ff": "Fulah",
+ "ffi": "Foia Foia",
+ "ffm": "Maasina Fulfulde",
+ "fgr": "Fongoro",
+ "fi": "Finnish",
+ "fia": "Nobiin",
+ "fie": "Fyer",
+ "fif": "Faifi",
+ "fil": "Filipino; Pilipino",
+ "fip": "Fipa",
+ "fir": "Firan",
+ "fit": "Tornedalen Finnish; Meänkieli",
+ "fiu": "Finno-Ugrian languages",
+ "fiw": "Fiwaga",
+ "fj": "Fijian",
+ "fkk": "Kirya-Konzəl",
+ "fkv": "Kven Finnish",
+ "fla": "Kalispel-Pend d'Oreille",
+ "flh": "Foau",
+ "fli": "Fali",
+ "fll": "North Fali",
+ "fln": "Flinders Island",
+ "flr": "Fuliiru",
+ "fly": "Flaaitaal; Tsotsitaal",
+ "fmp": "Fe'fe'",
+ "fmu": "Far Western Muria",
+ "fnb": "Fanbak",
+ "fng": "Fanagalo",
+ "fni": "Fania",
+ "fo": "Faroese",
+ "fod": "Foodo",
+ "foi": "Foi",
+ "fom": "Foma",
+ "fon": "Fon",
+ "for": "Fore",
+ "fos": "Siraya",
+ "fox": "Formosan languages",
+ "fpe": "Fernando Po Creole English",
+ "fqs": "Fas",
+ "fr": "French",
+ "frc": "Cajun French",
+ "frd": "Fordata",
+ "frk": "Frankish",
+ "frm": "Middle French (ca. 1400-1600)",
+ "fro": "Old French (842-ca. 1400)",
+ "frp": "Arpitan; Francoprovençal",
+ "frq": "Forak",
+ "frr": "Northern Frisian",
+ "frs": "Eastern Frisian",
+ "frt": "Fortsenal",
+ "fse": "Finnish Sign Language",
+ "fsl": "French Sign Language",
+ "fss": "Finland-Swedish Sign Language; finlandssvenskt teckenspråk; suomenruotsalainen viittomakieli",
+ "fub": "Adamawa Fulfulde",
+ "fuc": "Pulaar",
+ "fud": "East Futuna",
+ "fue": "Borgu Fulfulde",
+ "fuf": "Pular",
+ "fuh": "Western Niger Fulfulde",
+ "fui": "Bagirmi Fulfulde",
+ "fuj": "Ko",
+ "fum": "Fum",
+ "fun": "Fulniô",
+ "fuq": "Central-Eastern Niger Fulfulde",
+ "fur": "Friulian",
+ "fut": "Futuna-Aniwa",
+ "fuu": "Furu",
+ "fuv": "Nigerian Fulfulde",
+ "fuy": "Fuyug",
+ "fvr": "Fur",
+ "fwa": "Fwâi",
+ "fwe": "Fwe",
+ "fy": "Western Frisian",
+ "ga": "Irish",
+ "gaa": "Ga",
+ "gab": "Gabri",
+ "gac": "Mixed Great Andamanese",
+ "gad": "Gaddang",
+ "gae": "Guarequena",
+ "gaf": "Gende",
+ "gag": "Gagauz",
+ "gah": "Alekano",
+ "gai": "Borei",
+ "gaj": "Gadsup",
+ "gak": "Gamkonora",
+ "gal": "Galolen",
+ "gam": "Kandawo",
+ "gan": "Gan Chinese",
+ "gao": "Gants",
+ "gap": "Gal",
+ "gaq": "Gata'",
+ "gar": "Galeya",
+ "gas": "Adiwasi Garasia",
+ "gat": "Kenati",
+ "gau": "Mudhili Gadaba",
+ "gaw": "Nobonob",
+ "gax": "Borana-Arsi-Guji Oromo",
+ "gay": "Gayo",
+ "gaz": "West Central Oromo",
+ "gba": "Gbaya (Central African Republic)",
+ "gbb": "Kaytetye",
+ "gbd": "Karajarri",
+ "gbe": "Niksek",
+ "gbf": "Gaikundi",
+ "gbg": "Gbanziri",
+ "gbh": "Defi Gbe",
+ "gbi": "Galela",
+ "gbj": "Bodo Gadaba",
+ "gbk": "Gaddi",
+ "gbl": "Gamit",
+ "gbm": "Garhwali",
+ "gbn": "Mo'da",
+ "gbo": "Northern Grebo",
+ "gbp": "Gbaya-Bossangoa",
+ "gbq": "Gbaya-Bozoum",
+ "gbr": "Gbagyi",
+ "gbs": "Gbesi Gbe",
+ "gbu": "Gagadu",
+ "gbv": "Gbanu",
+ "gbw": "Gabi-Gabi",
+ "gbx": "Eastern Xwla Gbe",
+ "gby": "Gbari",
+ "gbz": "Zoroastrian Dari",
+ "gcc": "Mali",
+ "gcd": "Ganggalida",
+ "gce": "Galice",
+ "gcf": "Guadeloupean Creole French",
+ "gcl": "Grenadian Creole English",
+ "gcn": "Gaina",
+ "gcr": "Guianese Creole French",
+ "gct": "Colonia Tovar German",
+ "gd": "Scottish Gaelic; Gaelic",
+ "gda": "Gade Lohar",
+ "gdb": "Pottangi Ollar Gadaba",
+ "gdc": "Gugu Badhun",
+ "gdd": "Gedaged",
+ "gde": "Gude",
+ "gdf": "Guduf-Gava",
+ "gdg": "Ga'dang",
+ "gdh": "Gadjerawang; Gajirrabeng",
+ "gdi": "Gundi",
+ "gdj": "Gurdjar",
+ "gdk": "Gadang",
+ "gdl": "Dirasha",
+ "gdm": "Laal",
+ "gdn": "Umanakaina",
+ "gdo": "Ghodoberi",
+ "gdq": "Mehri",
+ "gdr": "Wipi",
+ "gds": "Ghandruk Sign Language",
+ "gdt": "Kungardutyi",
+ "gdu": "Gudu",
+ "gdx": "Godwari",
+ "gea": "Geruma",
+ "geb": "Kire",
+ "gec": "Gboloo Grebo",
+ "ged": "Gade",
+ "gef": "Gerai",
+ "geg": "Gengle",
+ "geh": "Hutterite German; Hutterisch",
+ "gei": "Gebe",
+ "gej": "Gen",
+ "gek": "Ywom",
+ "gel": "ut-Ma'in",
+ "gem": "Germanic languages",
+ "geq": "Geme",
+ "ges": "Geser-Gorom",
+ "gev": "Eviya",
+ "gew": "Gera",
+ "gex": "Garre",
+ "gey": "Enya",
+ "gez": "Geez",
+ "gfk": "Patpatar",
+ "gft": "Gafat",
+ "gga": "Gao",
+ "ggb": "Gbii",
+ "ggd": "Gugadj",
+ "gge": "Gurr-goni",
+ "ggg": "Gurgula",
+ "ggk": "Kungarakany",
+ "ggl": "Ganglau",
+ "ggt": "Gitua",
+ "ggu": "Gagu; Gban",
+ "ggw": "Gogodala",
+ "gha": "Ghadamès",
+ "ghc": "Hiberno-Scottish Gaelic",
+ "ghe": "Southern Ghale",
+ "ghh": "Northern Ghale",
+ "ghk": "Geko Karen",
+ "ghl": "Ghulfan",
+ "ghn": "Ghanongga",
+ "gho": "Ghomara",
+ "ghr": "Ghera",
+ "ghs": "Guhu-Samane",
+ "ght": "Kuke; Kutang Ghale",
+ "gia": "Kija",
+ "gib": "Gibanawa",
+ "gic": "Gail",
+ "gid": "Gidar",
+ "gie": "Gaɓogbo; Guébie",
+ "gig": "Goaria",
+ "gih": "Githabul",
+ "gii": "Girirra",
+ "gil": "Gilbertese",
+ "gim": "Gimi (Eastern Highlands)",
+ "gin": "Hinukh",
+ "gip": "Gimi (West New Britain)",
+ "giq": "Green Gelao",
+ "gir": "Red Gelao",
+ "gis": "North Giziga",
+ "git": "Gitxsan",
+ "giu": "Mulao",
+ "giw": "White Gelao",
+ "gix": "Gilima",
+ "giy": "Giyug",
+ "giz": "South Giziga",
+ "gjk": "Kachi Koli",
+ "gjm": "Gunditjmara",
+ "gjn": "Gonja",
+ "gjr": "Gurindji Kriol",
+ "gju": "Gujari",
+ "gka": "Guya",
+ "gkd": "Magɨ (Madang Province)",
+ "gke": "Ndai",
+ "gkn": "Gokana",
+ "gko": "Kok-Nar",
+ "gkp": "Guinea Kpelle",
+ "gku": "ǂUngkue",
+ "gl": "Galician",
+ "glb": "Belning",
+ "glc": "Bon Gula",
+ "gld": "Nanai",
+ "glh": "Northwest Pashai; Northwest Pashayi",
+ "glj": "Gula Iro",
+ "glk": "Gilaki",
+ "gll": "Garlali",
+ "glo": "Galambu",
+ "glr": "Glaro-Twabo",
+ "glu": "Gula (Chad)",
+ "glw": "Glavda",
+ "gly": "Gule",
+ "gma": "Gambera",
+ "gmb": "Gula'alaa",
+ "gmd": "Mághdì",
+ "gme": "East Germanic languages",
+ "gmg": "Magɨyi",
+ "gmh": "Middle High German (ca. 1050-1500)",
+ "gml": "Middle Low German",
+ "gmm": "Gbaya-Mbodomo",
+ "gmn": "Gimnime",
+ "gmq": "North Germanic languages",
+ "gmr": "Mirning; Mirniny",
+ "gmu": "Gumalu",
+ "gmv": "Gamo",
+ "gmw": "West Germanic languages",
+ "gmx": "Magoma",
+ "gmy": "Mycenaean Greek",
+ "gmz": "Mgbolizhia",
+ "gn": "Guarani",
+ "gna": "Kaansa",
+ "gnb": "Gangte",
+ "gnc": "Guanche",
+ "gnd": "Zulgo-Gemzek",
+ "gne": "Ganang",
+ "gng": "Ngangam",
+ "gnh": "Lere",
+ "gni": "Gooniyandi",
+ "gnj": "Ngen",
+ "gnk": "ǁGana",
+ "gnl": "Gangulu",
+ "gnm": "Ginuman",
+ "gnn": "Gumatj",
+ "gno": "Northern Gondi",
+ "gnq": "Gana",
+ "gnr": "Gureng Gureng",
+ "gnt": "Guntai",
+ "gnu": "Gnau",
+ "gnw": "Western Bolivian Guaraní",
+ "gnz": "Ganzi",
+ "goa": "Guro",
+ "gob": "Playero",
+ "goc": "Gorakor",
+ "god": "Godié",
+ "goe": "Gongduk",
+ "gof": "Gofa",
+ "gog": "Gogo",
+ "goh": "Old High German (ca. 750-1050)",
+ "goi": "Gobasi",
+ "goj": "Gowlan",
+ "gok": "Gowli",
+ "gol": "Gola",
+ "gom": "Goan Konkani",
+ "gon": "Gondi",
+ "goo": "Gone Dau",
+ "gop": "Yeretuar",
+ "goq": "Gorap",
+ "gor": "Gorontalo",
+ "gos": "Gronings",
+ "got": "Gothic",
+ "gou": "Gavar",
+ "gov": "Goo",
+ "gow": "Gorowa",
+ "gox": "Gobu",
+ "goy": "Goundo",
+ "goz": "Gozarkhani",
+ "gpa": "Gupa-Abawa",
+ "gpe": "Ghanaian Pidgin English",
+ "gpn": "Taiap",
+ "gqa": "Ga'anda",
+ "gqi": "Guiqiong",
+ "gqn": "Guana (Brazil)",
+ "gqr": "Gor",
+ "gqu": "Qau",
+ "gra": "Rajput Garasia",
+ "grb": "Grebo",
+ "grc": "Ancient Greek (to 1453)",
+ "grd": "Guruntum-Mbaaru",
+ "grg": "Madi",
+ "grh": "Gbiri-Niragu",
+ "gri": "Ghari",
+ "grj": "Southern Grebo",
+ "grk": "Greek languages",
+ "grm": "Kota Marudu Talantang",
+ "gro": "Groma",
+ "grq": "Gorovu",
+ "grr": "Taznatit",
+ "grs": "Gresi",
+ "grt": "Garo",
+ "gru": "Kistane",
+ "grv": "Central Grebo",
+ "grw": "Gweda",
+ "grx": "Guriaso",
+ "gry": "Barclayville Grebo",
+ "grz": "Guramalum",
+ "gse": "Ghanaian Sign Language",
+ "gsg": "German Sign Language",
+ "gsl": "Gusilay",
+ "gsm": "Guatemalan Sign Language",
+ "gsn": "Nema; Gusan",
+ "gso": "Southwest Gbaya",
+ "gsp": "Wasembo",
+ "gss": "Greek Sign Language",
+ "gsw": "Swiss German; Alemannic; Alsatian",
+ "gta": "Guató",
+ "gtu": "Aghu-Tharnggala",
+ "gu": "Gujarati",
+ "gua": "Shiki",
+ "gub": "Guajajára",
+ "guc": "Wayuu",
+ "gud": "Yocoboué Dida",
+ "gue": "Gurindji",
+ "guf": "Gupapuyngu",
+ "gug": "Paraguayan Guaraní",
+ "guh": "Guahibo",
+ "gui": "Eastern Bolivian Guaraní",
+ "guk": "Gumuz",
+ "gul": "Sea Island Creole English",
+ "gum": "Guambiano",
+ "gun": "Mbyá Guaraní",
+ "guo": "Guayabero",
+ "gup": "Gunwinggu",
+ "guq": "Aché",
+ "gur": "Farefare",
+ "gus": "Guinean Sign Language",
+ "gut": "Maléku Jaíka",
+ "guu": "Yanomamö",
+ "guw": "Gun",
+ "gux": "Gourmanchéma",
+ "guz": "Gusii; Ekegusii",
+ "gv": "Manx",
+ "gva": "Guana (Paraguay)",
+ "gvc": "Guanano",
+ "gve": "Duwet",
+ "gvf": "Golin",
+ "gvj": "Guajá",
+ "gvl": "Gulay",
+ "gvm": "Gurmana",
+ "gvn": "Kuku-Yalanji",
+ "gvo": "Gavião Do Jiparaná",
+ "gvp": "Pará Gavião",
+ "gvr": "Gurung",
+ "gvs": "Gumawana",
+ "gvy": "Guyani",
+ "gwa": "Mbato",
+ "gwb": "Gwa",
+ "gwc": "Gawri; Kalami",
+ "gwd": "Gawwada",
+ "gwe": "Gweno",
+ "gwf": "Gowro",
+ "gwg": "Moo",
+ "gwi": "Gwichʼin",
+ "gwj": "ǀGwi",
+ "gwm": "Awngthim",
+ "gwn": "Gwandara",
+ "gwr": "Gwere",
+ "gwt": "Gawar-Bati",
+ "gwu": "Guwamu",
+ "gww": "Kwini",
+ "gwx": "Gua",
+ "gxx": "Wè Southern",
+ "gya": "Northwest Gbaya",
+ "gyb": "Garus",
+ "gyd": "Kayardild",
+ "gye": "Gyem",
+ "gyf": "Gungabula",
+ "gyg": "Gbayi",
+ "gyi": "Gyele",
+ "gyl": "Gayil",
+ "gym": "Ngäbere",
+ "gyn": "Guyanese Creole English",
+ "gyo": "Gyalsumdo",
+ "gyr": "Guarayu",
+ "gyy": "Gunya",
+ "gyz": "Geji; Gyaazi",
+ "gza": "Ganza",
+ "gzi": "Gazi",
+ "gzn": "Gane",
+ "ha": "Hausa",
+ "haa": "Han",
+ "hab": "Hanoi Sign Language",
+ "hac": "Gurani",
+ "had": "Hatam",
+ "hae": "Eastern Oromo",
+ "haf": "Haiphong Sign Language",
+ "hag": "Hanga",
+ "hah": "Hahon",
+ "hai": "Haida",
+ "haj": "Hajong",
+ "hak": "Hakka Chinese",
+ "hal": "Halang",
+ "ham": "Hewa",
+ "han": "Hangaza",
+ "hao": "Hakö",
+ "hap": "Hupla",
+ "haq": "Ha",
+ "har": "Harari",
+ "has": "Haisla",
+ "hav": "Havu",
+ "haw": "Hawaiian",
+ "hax": "Southern Haida",
+ "hay": "Haya",
+ "haz": "Hazaragi",
+ "hba": "Hamba",
+ "hbb": "Huba",
+ "hbn": "Heiban",
+ "hbo": "Ancient Hebrew",
+ "hbu": "Habu",
+ "hca": "Andaman Creole Hindi",
+ "hch": "Huichol",
+ "hdn": "Northern Haida",
+ "hds": "Honduras Sign Language",
+ "hdy": "Hadiyya",
+ "he": "Hebrew",
+ "hea": "Northern Qiandong Miao",
+ "hed": "Herdé",
+ "heg": "Helong",
+ "heh": "Hehe",
+ "hei": "Heiltsuk",
+ "hem": "Hemba",
+ "hgm": "Haiǁom",
+ "hgw": "Haigwai",
+ "hhi": "Hoia Hoia",
+ "hhr": "Kerak",
+ "hhy": "Hoyahoya",
+ "hi": "Hindi",
+ "hia": "Lamang",
+ "hib": "Hibito",
+ "hid": "Hidatsa",
+ "hif": "Fiji Hindi",
+ "hig": "Kamwe",
+ "hih": "Pamosu",
+ "hii": "Hinduri",
+ "hij": "Hijuk",
+ "hik": "Seit-Kaitetu",
+ "hil": "Hiligaynon",
+ "him": "Himachali languages; Western Pahari languages",
+ "hio": "Tsoa",
+ "hir": "Himarimã",
+ "hit": "Hittite",
+ "hiw": "Hiw",
+ "hix": "Hixkaryána",
+ "hji": "Haji",
+ "hka": "Kahe",
+ "hke": "Hunde",
+ "hkh": "Khah; Poguli",
+ "hkk": "Hunjara-Kaina Ke",
+ "hkn": "Mel-Khaonh",
+ "hks": "Hong Kong Sign Language; Heung Kong Sau Yue",
+ "hla": "Halia",
+ "hlb": "Halbi",
+ "hld": "Halang Doan",
+ "hle": "Hlersu",
+ "hlt": "Matu Chin",
+ "hlu": "Hieroglyphic Luwian",
+ "hma": "Southern Mashan Hmong; Southern Mashan Miao",
+ "hmb": "Humburi Senni Songhay",
+ "hmc": "Central Huishui Hmong; Central Huishui Miao",
+ "hmd": "Large Flowery Miao; A-hmaos; Da-Hua Miao",
+ "hme": "Eastern Huishui Hmong; Eastern Huishui Miao",
+ "hmf": "Hmong Don",
+ "hmg": "Southwestern Guiyang Hmong",
+ "hmh": "Southwestern Huishui Hmong; Southwestern Huishui Miao",
+ "hmi": "Northern Huishui Hmong; Northern Huishui Miao",
+ "hmj": "Ge; Gejia",
+ "hmk": "Maek",
+ "hml": "Luopohe Hmong; Luopohe Miao",
+ "hmm": "Central Mashan Hmong; Central Mashan Miao",
+ "hmn": "Hmong; Mong",
+ "hmp": "Northern Mashan Hmong; Northern Mashan Miao",
+ "hmq": "Eastern Qiandong Miao",
+ "hmr": "Hmar",
+ "hms": "Southern Qiandong Miao",
+ "hmt": "Hamtai",
+ "hmu": "Hamap",
+ "hmv": "Hmong Dô",
+ "hmw": "Western Mashan Hmong; Western Mashan Miao",
+ "hmx": "Hmong-Mien languages",
+ "hmy": "Southern Guiyang Hmong; Southern Guiyang Miao",
+ "hmz": "Hmong Shua; Sinicized Miao",
+ "hna": "Mina (Cameroon)",
+ "hnd": "Southern Hindko",
+ "hne": "Chhattisgarhi",
+ "hng": "Hungu",
+ "hnh": "ǁAni",
+ "hni": "Hani",
+ "hnj": "Hmong Njua; Mong Leng; Mong Njua",
+ "hnn": "Hanunoo",
+ "hno": "Northern Hindko",
+ "hns": "Caribbean Hindustani",
+ "hnu": "Hung",
+ "ho": "Hiri Motu",
+ "hoa": "Hoava",
+ "hob": "Mari (Madang Province)",
+ "hoc": "Ho",
+ "hod": "Holma",
+ "hoe": "Horom",
+ "hoh": "Hobyót",
+ "hoi": "Holikachuk",
+ "hoj": "Hadothi; Haroti",
+ "hok": "Hokan languages",
+ "hol": "Holu",
+ "hom": "Homa",
+ "hoo": "Holoholo",
+ "hop": "Hopi",
+ "hor": "Horo",
+ "hos": "Ho Chi Minh City Sign Language",
+ "hot": "Hote; Malê",
+ "hov": "Hovongan",
+ "how": "Honi",
+ "hoy": "Holiya",
+ "hoz": "Hozo",
+ "hpo": "Hpon",
+ "hps": "Hawai'i Sign Language (HSL); Hawai'i Pidgin Sign Language",
+ "hr": "Croatian",
+ "hra": "Hrangkhol",
+ "hrc": "Niwer Mil",
+ "hre": "Hre",
+ "hrk": "Haruku",
+ "hrm": "Horned Miao",
+ "hro": "Haroi",
+ "hrp": "Nhirrpi",
+ "hrt": "Hértevin",
+ "hru": "Hruso",
+ "hrw": "Warwar Feni",
+ "hrx": "Hunsrik",
+ "hrz": "Harzani",
+ "hsb": "Upper Sorbian",
+ "hsh": "Hungarian Sign Language",
+ "hsl": "Hausa Sign Language",
+ "hsn": "Xiang Chinese",
+ "hss": "Harsusi",
+ "ht": "Haitian; Haitian Creole",
+ "hti": "Hoti",
+ "hto": "Minica Huitoto",
+ "hts": "Hadza",
+ "htu": "Hitu",
+ "htx": "Middle Hittite",
+ "hu": "Hungarian",
+ "hub": "Huambisa",
+ "huc": "ǂHua; ǂʼAmkhoe",
+ "hud": "Huaulu",
+ "hue": "San Francisco Del Mar Huave",
+ "huf": "Humene",
+ "hug": "Huachipaeri",
+ "huh": "Huilliche",
+ "hui": "Huli",
+ "huj": "Northern Guiyang Hmong; Northern Guiyang Miao",
+ "huk": "Hulung",
+ "hul": "Hula",
+ "hum": "Hungana",
+ "huo": "Hu",
+ "hup": "Hupa",
+ "huq": "Tsat",
+ "hur": "Halkomelem",
+ "hus": "Huastec",
+ "hut": "Humla",
+ "huu": "Murui Huitoto",
+ "huv": "San Mateo Del Mar Huave",
+ "huw": "Hukumina",
+ "hux": "Nüpode Huitoto",
+ "huy": "Hulaulá",
+ "huz": "Hunzib",
+ "hvc": "Haitian Vodoun Culture Language",
+ "hve": "San Dionisio Del Mar Huave",
+ "hvk": "Haveke",
+ "hvn": "Sabu",
+ "hvv": "Santa María Del Mar Huave",
+ "hwa": "Wané",
+ "hwc": "Hawai'i Creole English; Hawai'i Pidgin",
+ "hwo": "Hwana",
+ "hy": "Armenian",
+ "hya": "Hya",
+ "hyw": "Western Armenian",
+ "hyx": "Armenian (family)",
+ "hz": "Herero",
+ "ia": "Interlingua (International Auxiliary Language Association)",
+ "iai": "Iaai",
+ "ian": "Iatmul",
+ "iar": "Purari",
+ "iba": "Iban",
+ "ibb": "Ibibio",
+ "ibd": "Iwaidja",
+ "ibe": "Akpes",
+ "ibg": "Ibanag",
+ "ibh": "Bih",
+ "ibl": "Ibaloi",
+ "ibm": "Agoi",
+ "ibn": "Ibino",
+ "ibr": "Ibuoro",
+ "ibu": "Ibu",
+ "iby": "Ibani",
+ "ica": "Ede Ica",
+ "ich": "Etkywan",
+ "icl": "Icelandic Sign Language",
+ "icr": "Islander Creole English",
+ "id": "Indonesian",
+ "ida": "Idakho-Isukha-Tiriki; Luidakho-Luisukha-Lutirichi",
+ "idb": "Indo-Portuguese",
+ "idc": "Idon; Ajiya",
+ "idd": "Ede Idaca",
+ "ide": "Idere",
+ "idi": "Idi",
+ "idr": "Indri",
+ "ids": "Idesa",
+ "idt": "Idaté",
+ "idu": "Idoma",
+ "ie": "Interlingue; Occidental",
+ "ifa": "Amganad Ifugao",
+ "ifb": "Batad Ifugao; Ayangan Ifugao",
+ "ife": "Ifè",
+ "iff": "Ifo",
+ "ifk": "Tuwali Ifugao",
+ "ifm": "Teke-Fuumu",
+ "ifu": "Mayoyao Ifugao",
+ "ify": "Keley-I Kallahan",
+ "ig": "Igbo",
+ "igb": "Ebira",
+ "ige": "Igede",
+ "igg": "Igana",
+ "igl": "Igala",
+ "igm": "Kanggape",
+ "ign": "Ignaciano",
+ "igo": "Isebe",
+ "igs": "Interglossa",
+ "igw": "Igwe",
+ "ihb": "Iha Based Pidgin",
+ "ihi": "Ihievbe",
+ "ihp": "Iha",
+ "ihw": "Bidhawal",
+ "ii": "Sichuan Yi; Nuosu",
+ "iin": "Thiin",
+ "iir": "Indo-Iranian languages",
+ "ijc": "Izon",
+ "ije": "Biseni",
+ "ijj": "Ede Ije",
+ "ijn": "Kalabari",
+ "ijo": "Ijo languages",
+ "ijs": "Southeast Ijo",
+ "ik": "Inupiaq",
+ "ike": "Eastern Canadian Inuktitut",
+ "iki": "Iko",
+ "ikk": "Ika",
+ "ikl": "Ikulu",
+ "iko": "Olulumo-Ikom",
+ "ikp": "Ikpeshi",
+ "ikr": "Ikaranggal",
+ "iks": "Inuit Sign Language",
+ "ikt": "Inuinnaqtun; Western Canadian Inuktitut",
+ "ikv": "Iku-Gora-Ankwa",
+ "ikw": "Ikwere",
+ "ikx": "Ik",
+ "ikz": "Ikizu",
+ "ila": "Ile Ape",
+ "ilb": "Ila",
+ "ilg": "Garig-Ilgar",
+ "ili": "Ili Turki",
+ "ilk": "Ilongot",
+ "ilm": "Iranun (Malaysia)",
+ "ilo": "Iloko",
+ "ilp": "Iranun (Philippines)",
+ "ils": "International Sign",
+ "ilu": "Ili'uun",
+ "ilv": "Ilue",
+ "ima": "Mala Malasar",
+ "imi": "Anamgura",
+ "iml": "Miluk",
+ "imn": "Imonda",
+ "imo": "Imbongu",
+ "imr": "Imroing",
+ "ims": "Marsian",
+ "imt": "Imotong",
+ "imy": "Milyan",
+ "inb": "Inga",
+ "inc": "Indic languages",
+ "ine": "Indo-European languages",
+ "ing": "Degexit'an",
+ "inh": "Ingush",
+ "inj": "Jungle Inga",
+ "inl": "Indonesian Sign Language",
+ "inm": "Minaean",
+ "inn": "Isinai",
+ "ino": "Inoke-Yate",
+ "inp": "Iñapari",
+ "ins": "Indian Sign Language",
+ "int": "Intha",
+ "inz": "Ineseño",
+ "io": "Ido",
+ "ior": "Inor",
+ "iou": "Tuma-Irumu",
+ "iow": "Iowa-Oto",
+ "ipi": "Ipili",
+ "ipo": "Ipiko",
+ "iqu": "Iquito",
+ "iqw": "Ikwo",
+ "ira": "Iranian languages",
+ "ire": "Iresim",
+ "irh": "Irarutu",
+ "iri": "Rigwe; Irigwe",
+ "irk": "Iraqw",
+ "irn": "Irántxe",
+ "iro": "Iroquoian languages",
+ "irr": "Ir",
+ "iru": "Irula",
+ "irx": "Kamberau",
+ "iry": "Iraya",
+ "is": "Icelandic",
+ "isa": "Isabi",
+ "isc": "Isconahua",
+ "isd": "Isnag",
+ "ise": "Italian Sign Language",
+ "isg": "Irish Sign Language",
+ "ish": "Esan",
+ "isi": "Nkem-Nkum",
+ "isk": "Ishkashimi",
+ "ism": "Masimasi",
+ "isn": "Isanzu",
+ "iso": "Isoko",
+ "isr": "Israeli Sign Language",
+ "ist": "Istriot",
+ "isu": "Isu (Menchum Division)",
+ "it": "Italian",
+ "itb": "Binongan Itneg",
+ "itc": "Italic languages",
+ "itd": "Southern Tidung",
+ "ite": "Itene",
+ "iti": "Inlaod Itneg",
+ "itk": "Judeo-Italian",
+ "itl": "Itelmen",
+ "itm": "Itu Mbon Uzo",
+ "ito": "Itonama",
+ "itr": "Iteri",
+ "its": "Isekiri",
+ "itt": "Maeng Itneg",
+ "itv": "Itawit",
+ "itw": "Ito",
+ "itx": "Itik",
+ "ity": "Moyadan Itneg",
+ "itz": "Itzá",
+ "iu": "Inuktitut",
+ "ium": "Iu Mien",
+ "ivb": "Ibatan",
+ "ivv": "Ivatan",
+ "iwk": "I-Wak",
+ "iwm": "Iwam",
+ "iwo": "Iwur",
+ "iws": "Sepik Iwam",
+ "ixc": "Ixcatec",
+ "ixl": "Ixil",
+ "iya": "Iyayu",
+ "iyo": "Mesaka",
+ "iyx": "Yaka (Congo)",
+ "izh": "Ingrian",
+ "izr": "Izere",
+ "izz": "Izii",
+ "ja": "Japanese",
+ "jaa": "Jamamadí",
+ "jab": "Hyam",
+ "jac": "Popti'; Jakalteko",
+ "jad": "Jahanka",
+ "jae": "Yabem",
+ "jaf": "Jara",
+ "jah": "Jah Hut",
+ "jaj": "Zazao",
+ "jak": "Jakun",
+ "jal": "Yalahatan",
+ "jam": "Jamaican Creole English",
+ "jan": "Jandai",
+ "jao": "Yanyuwa",
+ "jaq": "Yaqay",
+ "jas": "New Caledonian Javanese",
+ "jat": "Jakati",
+ "jau": "Yaur",
+ "jax": "Jambi Malay",
+ "jay": "Yan-nhangu; Nhangu",
+ "jaz": "Jawe",
+ "jbe": "Judeo-Berber",
+ "jbi": "Badjiri",
+ "jbj": "Arandai",
+ "jbk": "Barikewa",
+ "jbm": "Bijim",
+ "jbn": "Nafusi",
+ "jbo": "Lojban",
+ "jbr": "Jofotek-Bromnya",
+ "jbt": "Jabutí",
+ "jbu": "Jukun Takum",
+ "jbw": "Yawijibaya",
+ "jcs": "Jamaican Country Sign Language",
+ "jct": "Krymchak",
+ "jda": "Jad",
+ "jdg": "Jadgali",
+ "jdt": "Judeo-Tat",
+ "jeb": "Jebero",
+ "jee": "Jerung",
+ "jeh": "Jeh",
+ "jei": "Yei",
+ "jek": "Jeri Kuo",
+ "jel": "Yelmek",
+ "jen": "Dza",
+ "jer": "Jere",
+ "jet": "Manem",
+ "jeu": "Jonkor Bourmataguil",
+ "jgb": "Ngbee",
+ "jge": "Judeo-Georgian",
+ "jgk": "Gwak",
+ "jgo": "Ngomba",
+ "jhi": "Jehai",
+ "jhs": "Jhankot Sign Language",
+ "jia": "Jina",
+ "jib": "Jibu",
+ "jic": "Tol",
+ "jid": "Bu (Kaduna State)",
+ "jie": "Jilbe",
+ "jig": "Jingulu; Djingili",
+ "jih": "sTodsde; Shangzhai",
+ "jii": "Jiiddu",
+ "jil": "Jilim",
+ "jim": "Jimi (Cameroon)",
+ "jio": "Jiamao",
+ "jiq": "Guanyinqiao; Lavrung",
+ "jit": "Jita",
+ "jiu": "Youle Jinuo",
+ "jiv": "Shuar",
+ "jiy": "Buyuan Jinuo",
+ "jje": "Jejueo",
+ "jjr": "Bankal",
+ "jka": "Kaera",
+ "jkm": "Mobwa Karen",
+ "jko": "Kubo",
+ "jkp": "Paku Karen",
+ "jkr": "Koro (India)",
+ "jks": "Amami Koniya Sign Language",
+ "jku": "Labir",
+ "jle": "Ngile",
+ "jls": "Jamaican Sign Language",
+ "jma": "Dima",
+ "jmb": "Zumbun",
+ "jmc": "Machame",
+ "jmd": "Yamdena",
+ "jmi": "Jimi (Nigeria)",
+ "jml": "Jumli",
+ "jmn": "Makuri Naga",
+ "jmr": "Kamara",
+ "jms": "Mashi (Nigeria)",
+ "jmw": "Mouwase",
+ "jmx": "Western Juxtlahuaca Mixtec",
+ "jna": "Jangshung",
+ "jnd": "Jandavra",
+ "jng": "Yangman",
+ "jni": "Janji",
+ "jnj": "Yemsa",
+ "jnl": "Rawat",
+ "jns": "Jaunsari",
+ "job": "Joba",
+ "jod": "Wojenaka",
+ "jog": "Jogi",
+ "jor": "Jorá",
+ "jos": "Jordanian Sign Language",
+ "jow": "Jowulu",
+ "jpa": "Jewish Palestinian Aramaic",
+ "jpr": "Judeo-Persian",
+ "jpx": "Japanese (family)",
+ "jqr": "Jaqaru",
+ "jra": "Jarai",
+ "jrb": "Judeo-Arabic",
+ "jrr": "Jiru",
+ "jrt": "Jakattoe",
+ "jru": "Japrería",
+ "jsl": "Japanese Sign Language",
+ "jua": "Júma",
+ "jub": "Wannu",
+ "juc": "Jurchen",
+ "jud": "Worodougou",
+ "juh": "Hõne",
+ "jui": "Ngadjuri",
+ "juk": "Wapan",
+ "jul": "Jirel",
+ "jum": "Jumjum",
+ "jun": "Juang",
+ "juo": "Jiba",
+ "jup": "Hupdë",
+ "jur": "Jurúna",
+ "jus": "Jumla Sign Language",
+ "jut": "Jutish",
+ "juu": "Ju",
+ "juw": "Wãpha",
+ "juy": "Juray",
+ "jv": "Javanese",
+ "jvd": "Javindo",
+ "jvn": "Caribbean Javanese",
+ "jwi": "Jwira-Pepesa",
+ "jya": "Jiarong",
+ "jye": "Judeo-Yemeni Arabic",
+ "jyy": "Jaya",
+ "ka": "Georgian",
+ "kaa": "Kara-Kalpak; Karakalpak",
+ "kab": "Kabyle",
+ "kac": "Kachin; Jingpho",
+ "kad": "Adara",
+ "kae": "Ketangalan",
+ "kaf": "Katso",
+ "kag": "Kajaman",
+ "kah": "Kara (Central African Republic)",
+ "kai": "Karekare",
+ "kaj": "Jju",
+ "kak": "Kalanguya; Kayapa Kallahan",
+ "kam": "Kamba (Kenya)",
+ "kao": "Xaasongaxango",
+ "kap": "Bezhta",
+ "kaq": "Capanahua",
+ "kar": "Karen languages",
+ "kav": "Katukína",
+ "kaw": "Kawi",
+ "kax": "Kao",
+ "kay": "Kamayurá",
+ "kba": "Kalarko",
+ "kbb": "Kaxuiâna",
+ "kbc": "Kadiwéu",
+ "kbd": "Kabardian",
+ "kbe": "Kanju",
+ "kbg": "Khamba",
+ "kbh": "Camsá",
+ "kbi": "Kaptiau",
+ "kbj": "Kari",
+ "kbk": "Grass Koiari",
+ "kbl": "Kanembu",
+ "kbm": "Iwal",
+ "kbn": "Kare (Central African Republic)",
+ "kbo": "Keliko",
+ "kbp": "Kabiyè",
+ "kbq": "Kamano",
+ "kbr": "Kafa",
+ "kbs": "Kande",
+ "kbt": "Abadi",
+ "kbu": "Kabutra",
+ "kbv": "Dera (Indonesia)",
+ "kbw": "Kaiep",
+ "kbx": "Ap Ma",
+ "kby": "Manga Kanuri",
+ "kbz": "Duhwa",
+ "kca": "Khanty",
+ "kcb": "Kawacha",
+ "kcc": "Lubila",
+ "kcd": "Ngkâlmpw Kanum",
+ "kce": "Kaivi",
+ "kcf": "Ukaan",
+ "kcg": "Tyap",
+ "kch": "Vono",
+ "kci": "Kamantan",
+ "kcj": "Kobiana",
+ "kck": "Kalanga",
+ "kcl": "Kela (Papua New Guinea); Kala",
+ "kcm": "Gula (Central African Republic)",
+ "kcn": "Nubi",
+ "kco": "Kinalakna",
+ "kcp": "Kanga",
+ "kcq": "Kamo",
+ "kcr": "Katla",
+ "kcs": "Koenoem",
+ "kct": "Kaian",
+ "kcu": "Kami (Tanzania)",
+ "kcv": "Kete",
+ "kcw": "Kabwari",
+ "kcx": "Kachama-Ganjule",
+ "kcy": "Korandje",
+ "kcz": "Konongo",
+ "kda": "Worimi",
+ "kdc": "Kutu",
+ "kdd": "Yankunytjatjara",
+ "kde": "Makonde",
+ "kdf": "Mamusi",
+ "kdg": "Seba",
+ "kdh": "Tem",
+ "kdi": "Kumam",
+ "kdj": "Karamojong",
+ "kdk": "Numèè; Kwényi",
+ "kdl": "Tsikimba",
+ "kdm": "Kagoma",
+ "kdn": "Kunda",
+ "kdo": "Kordofanian languages",
+ "kdp": "Kaningdon-Nindem",
+ "kdq": "Koch",
+ "kdr": "Karaim",
+ "kdt": "Kuy",
+ "kdu": "Kadaru",
+ "kdw": "Koneraw",
+ "kdx": "Kam",
+ "kdy": "Keder; Keijar",
+ "kdz": "Kwaja",
+ "kea": "Kabuverdianu",
+ "keb": "Kélé",
+ "kec": "Keiga",
+ "ked": "Kerewe",
+ "kee": "Eastern Keres",
+ "kef": "Kpessi",
+ "keg": "Tese",
+ "keh": "Keak",
+ "kei": "Kei",
+ "kej": "Kadar",
+ "kek": "Kekchí",
+ "kel": "Kela (Democratic Republic of Congo)",
+ "kem": "Kemak",
+ "ken": "Kenyang",
+ "keo": "Kakwa",
+ "kep": "Kaikadi",
+ "keq": "Kamar",
+ "ker": "Kera",
+ "kes": "Kugbo",
+ "ket": "Ket",
+ "keu": "Akebu",
+ "kev": "Kanikkaran",
+ "kew": "West Kewa",
+ "kex": "Kukna",
+ "key": "Kupia",
+ "kez": "Kukele",
+ "kfa": "Kodava",
+ "kfb": "Northwestern Kolami",
+ "kfc": "Konda-Dora",
+ "kfd": "Korra Koraga",
+ "kfe": "Kota (India)",
+ "kff": "Koya",
+ "kfg": "Kudiya",
+ "kfh": "Kurichiya",
+ "kfi": "Kannada Kurumba",
+ "kfj": "Kemiehua",
+ "kfk": "Kinnauri",
+ "kfl": "Kung",
+ "kfm": "Khunsari",
+ "kfn": "Kuk",
+ "kfo": "Koro (Côte d'Ivoire)",
+ "kfp": "Korwa",
+ "kfq": "Korku",
+ "kfr": "Kachhi; Kutchi",
+ "kfs": "Bilaspuri",
+ "kft": "Kanjari",
+ "kfu": "Katkari",
+ "kfv": "Kurmukar",
+ "kfw": "Kharam Naga",
+ "kfx": "Kullu Pahari",
+ "kfy": "Kumaoni",
+ "kfz": "Koromfé",
+ "kg": "Kongo",
+ "kga": "Koyaga",
+ "kgb": "Kawe",
+ "kge": "Komering",
+ "kgf": "Kube",
+ "kgg": "Kusunda",
+ "kgi": "Selangor Sign Language",
+ "kgj": "Gamale Kham",
+ "kgk": "Kaiwá",
+ "kgl": "Kunggari",
+ "kgm": "Karipúna",
+ "kgn": "Karingani",
+ "kgo": "Krongo",
+ "kgp": "Kaingang",
+ "kgq": "Kamoro",
+ "kgr": "Abun",
+ "kgs": "Kumbainggar",
+ "kgt": "Somyev",
+ "kgu": "Kobol",
+ "kgv": "Karas",
+ "kgw": "Karon Dori",
+ "kgx": "Kamaru",
+ "kgy": "Kyerung",
+ "kha": "Khasi",
+ "khb": "Lü",
+ "khc": "Tukang Besi North",
+ "khd": "Bädi Kanum",
+ "khe": "Korowai",
+ "khf": "Khuen",
+ "khg": "Khams Tibetan",
+ "khh": "Kehu",
+ "khi": "Khoisan languages",
+ "khj": "Kuturmi",
+ "khk": "Halh Mongolian",
+ "khl": "Lusi",
+ "khn": "Khandesi",
+ "kho": "Khotanese; Sakan",
+ "khp": "Kapori; Kapauri",
+ "khq": "Koyra Chiini Songhay",
+ "khr": "Kharia",
+ "khs": "Kasua",
+ "kht": "Khamti",
+ "khu": "Nkhumbi",
+ "khv": "Khvarshi",
+ "khw": "Khowar",
+ "khx": "Kanu",
+ "khy": "Kele (Democratic Republic of Congo)",
+ "khz": "Keapara",
+ "ki": "Kikuyu; Gikuyu",
+ "kia": "Kim",
+ "kib": "Koalib",
+ "kic": "Kickapoo",
+ "kid": "Koshin",
+ "kie": "Kibet",
+ "kif": "Eastern Parbate Kham",
+ "kig": "Kimaama; Kimaghima",
+ "kih": "Kilmeri",
+ "kii": "Kitsai",
+ "kij": "Kilivila",
+ "kil": "Kariya",
+ "kim": "Karagas",
+ "kio": "Kiowa",
+ "kip": "Sheshi Kham",
+ "kiq": "Kosadle; Kosare",
+ "kis": "Kis",
+ "kit": "Agob",
+ "kiu": "Kirmanjki (individual language)",
+ "kiv": "Kimbu",
+ "kiw": "Northeast Kiwai",
+ "kix": "Khiamniungan Naga",
+ "kiy": "Kirikiri",
+ "kiz": "Kisi",
+ "kj": "Kuanyama; Kwanyama",
+ "kja": "Mlap",
+ "kjb": "Q'anjob'al; Kanjobal",
+ "kjc": "Coastal Konjo",
+ "kjd": "Southern Kiwai",
+ "kje": "Kisar",
+ "kjg": "Khmu",
+ "kjh": "Khakas",
+ "kji": "Zabana",
+ "kjj": "Khinalugh",
+ "kjk": "Highland Konjo",
+ "kjl": "Western Parbate Kham",
+ "kjm": "Kháng",
+ "kjn": "Kunjen",
+ "kjo": "Harijan Kinnauri",
+ "kjp": "Pwo Eastern Karen",
+ "kjq": "Western Keres",
+ "kjr": "Kurudu",
+ "kjs": "East Kewa",
+ "kjt": "Phrae Pwo Karen",
+ "kju": "Kashaya",
+ "kjv": "Kaikavian Literary Language",
+ "kjx": "Ramopa",
+ "kjy": "Erave",
+ "kjz": "Bumthangkha",
+ "kk": "Kazakh",
+ "kka": "Kakanda",
+ "kkb": "Kwerisa",
+ "kkc": "Odoodee",
+ "kkd": "Kinuku",
+ "kke": "Kakabe",
+ "kkf": "Kalaktang Monpa",
+ "kkg": "Mabaka Valley Kalinga",
+ "kkh": "Khün",
+ "kki": "Kagulu",
+ "kkj": "Kako",
+ "kkk": "Kokota",
+ "kkl": "Kosarek Yale",
+ "kkm": "Kiong",
+ "kkn": "Kon Keu",
+ "kko": "Karko",
+ "kkp": "Gugubera; Koko-Bera",
+ "kkq": "Kaeku",
+ "kkr": "Kir-Balar",
+ "kks": "Giiwo",
+ "kkt": "Koi",
+ "kku": "Tumi",
+ "kkv": "Kangean",
+ "kkw": "Teke-Kukuya",
+ "kkx": "Kohin",
+ "kky": "Guugu Yimidhirr; Guguyimidjir",
+ "kkz": "Kaska",
+ "kl": "Kalaallisut; Greenlandic",
+ "kla": "Klamath-Modoc",
+ "klb": "Kiliwa",
+ "klc": "Kolbila",
+ "kld": "Gamilaraay",
+ "kle": "Kulung (Nepal)",
+ "klf": "Kendeje",
+ "klg": "Tagakaulo",
+ "klh": "Weliki",
+ "kli": "Kalumpang",
+ "klj": "Khalaj",
+ "klk": "Kono (Nigeria)",
+ "kll": "Kagan Kalagan",
+ "klm": "Migum",
+ "kln": "Kalenjin",
+ "klo": "Kapya",
+ "klp": "Kamasa",
+ "klq": "Rumu",
+ "klr": "Khaling",
+ "kls": "Kalasha",
+ "klt": "Nukna",
+ "klu": "Klao",
+ "klv": "Maskelynes",
+ "klw": "Tado; Lindu",
+ "klx": "Koluwawa",
+ "kly": "Kalao",
+ "klz": "Kabola",
+ "km": "Khmer; Central Khmer",
+ "kma": "Konni",
+ "kmb": "Kimbundu",
+ "kmc": "Southern Dong",
+ "kmd": "Majukayang Kalinga",
+ "kme": "Bakole",
+ "kmf": "Kare (Papua New Guinea)",
+ "kmg": "Kâte",
+ "kmh": "Kalam",
+ "kmi": "Kami (Nigeria)",
+ "kmj": "Kumarbhag Paharia",
+ "kmk": "Limos Kalinga",
+ "kml": "Tanudan Kalinga",
+ "kmm": "Kom (India)",
+ "kmn": "Awtuw",
+ "kmo": "Kwoma",
+ "kmp": "Gimme",
+ "kmq": "Kwama",
+ "kmr": "Northern Kurdish",
+ "kms": "Kamasau",
+ "kmt": "Kemtuik",
+ "kmu": "Kanite",
+ "kmv": "Karipúna Creole French",
+ "kmw": "Komo (Democratic Republic of Congo)",
+ "kmx": "Waboda",
+ "kmy": "Koma",
+ "kmz": "Khorasani Turkish",
+ "kn": "Kannada",
+ "kna": "Dera (Nigeria)",
+ "knb": "Lubuagan Kalinga",
+ "knc": "Central Kanuri",
+ "knd": "Konda",
+ "kne": "Kankanaey",
+ "knf": "Mankanya",
+ "kng": "Koongo",
+ "kni": "Kanufi",
+ "knj": "Western Kanjobal",
+ "knk": "Kuranko",
+ "knl": "Keninjal",
+ "knm": "Kanamarí",
+ "knn": "Konkani (individual language)",
+ "kno": "Kono (Sierra Leone)",
+ "knp": "Kwanja",
+ "knq": "Kintaq",
+ "knr": "Kaningra",
+ "kns": "Kensiu",
+ "knt": "Panoan Katukína",
+ "knu": "Kono (Guinea)",
+ "knv": "Tabo",
+ "knw": "Kung-Ekoka",
+ "knx": "Kendayan; Salako",
+ "kny": "Kanyok",
+ "knz": "Kalamsé",
+ "ko": "Korean",
+ "koa": "Konomala",
+ "koc": "Kpati",
+ "kod": "Kodi",
+ "koe": "Kacipo-Bale Suri",
+ "kof": "Kubi",
+ "kog": "Cogui; Kogi",
+ "koh": "Koyo",
+ "koi": "Komi-Permyak",
+ "kok": "Konkani (macrolanguage)",
+ "kol": "Kol (Papua New Guinea)",
+ "koo": "Konzo",
+ "kop": "Waube",
+ "koq": "Kota (Gabon)",
+ "kos": "Kosraean",
+ "kot": "Lagwan",
+ "kou": "Koke",
+ "kov": "Kudu-Camo",
+ "kow": "Kugama",
+ "koy": "Koyukon",
+ "koz": "Korak",
+ "kpa": "Kutto",
+ "kpb": "Mullu Kurumba",
+ "kpc": "Curripaco",
+ "kpd": "Koba",
+ "kpe": "Kpelle",
+ "kpf": "Komba",
+ "kpg": "Kapingamarangi",
+ "kph": "Kplang",
+ "kpi": "Kofei",
+ "kpj": "Karajá",
+ "kpk": "Kpan",
+ "kpl": "Kpala",
+ "kpm": "Koho",
+ "kpn": "Kepkiriwát",
+ "kpo": "Ikposo",
+ "kpq": "Korupun-Sela",
+ "kpr": "Korafe-Yegha",
+ "kps": "Tehit",
+ "kpt": "Karata",
+ "kpu": "Kafoa",
+ "kpv": "Komi-Zyrian",
+ "kpw": "Kobon",
+ "kpx": "Mountain Koiali",
+ "kpy": "Koryak",
+ "kpz": "Kupsabiny",
+ "kqa": "Mum",
+ "kqb": "Kovai",
+ "kqc": "Doromu-Koki",
+ "kqd": "Koy Sanjaq Surat",
+ "kqe": "Kalagan",
+ "kqf": "Kakabai",
+ "kqg": "Khe",
+ "kqh": "Kisankasa",
+ "kqi": "Koitabu",
+ "kqj": "Koromira",
+ "kqk": "Kotafon Gbe",
+ "kql": "Kyenele",
+ "kqm": "Khisa",
+ "kqn": "Kaonde",
+ "kqo": "Eastern Krahn",
+ "kqp": "Kimré",
+ "kqq": "Krenak",
+ "kqr": "Kimaragang",
+ "kqs": "Northern Kissi",
+ "kqt": "Klias River Kadazan",
+ "kqu": "Seroa",
+ "kqv": "Okolod",
+ "kqw": "Kandas",
+ "kqx": "Mser",
+ "kqy": "Koorete",
+ "kqz": "Korana",
+ "kr": "Kanuri",
+ "kra": "Kumhali",
+ "krb": "Karkin",
+ "krc": "Karachay-Balkar",
+ "krd": "Kairui-Midiki",
+ "kre": "Panará",
+ "krf": "Koro (Vanuatu)",
+ "krh": "Kurama",
+ "kri": "Krio",
+ "krj": "Kinaray-A",
+ "krk": "Kerek",
+ "krl": "Karelian",
+ "krn": "Sapo",
+ "kro": "Kru languages",
+ "krp": "Korop",
+ "krr": "Krung",
+ "krs": "Gbaya (Sudan)",
+ "krt": "Tumari Kanuri",
+ "kru": "Kurukh",
+ "krv": "Kavet",
+ "krw": "Western Krahn",
+ "krx": "Karon",
+ "kry": "Kryts",
+ "krz": "Sota Kanum",
+ "ks": "Kashmiri",
+ "ksa": "Shuwa-Zamani",
+ "ksb": "Shambala",
+ "ksc": "Southern Kalinga",
+ "ksd": "Kuanua",
+ "kse": "Kuni",
+ "ksf": "Bafia",
+ "ksg": "Kusaghe",
+ "ksh": "Kölsch",
+ "ksi": "Krisa; I'saka",
+ "ksj": "Uare",
+ "ksk": "Kansa",
+ "ksl": "Kumalu",
+ "ksm": "Kumba",
+ "ksn": "Kasiguranin",
+ "kso": "Kofa",
+ "ksp": "Kaba",
+ "ksq": "Kwaami",
+ "ksr": "Borong",
+ "kss": "Southern Kisi",
+ "kst": "Winyé",
+ "ksu": "Khamyang",
+ "ksv": "Kusu",
+ "ksw": "S'gaw Karen",
+ "ksx": "Kedang",
+ "ksy": "Kharia Thar",
+ "ksz": "Kodaku",
+ "kta": "Katua",
+ "ktb": "Kambaata",
+ "ktc": "Kholok",
+ "ktd": "Kokata; Kukatha",
+ "kte": "Nubri",
+ "ktf": "Kwami",
+ "ktg": "Kalkutung",
+ "kth": "Karanga",
+ "kti": "North Muyu",
+ "ktj": "Plapo Krumen",
+ "ktk": "Kaniet",
+ "ktl": "Koroshi",
+ "ktm": "Kurti",
+ "ktn": "Karitiâna",
+ "kto": "Kuot",
+ "ktp": "Kaduo",
+ "ktq": "Katabaga",
+ "kts": "South Muyu",
+ "ktt": "Ketum",
+ "ktu": "Kituba (Democratic Republic of Congo)",
+ "ktv": "Eastern Katu",
+ "ktw": "Kato",
+ "ktx": "Kaxararí",
+ "kty": "Kango (Bas-Uélé District)",
+ "ktz": "Juǀʼhoan; Juǀʼhoansi",
+ "ku": "Kurdish",
+ "kub": "Kutep",
+ "kuc": "Kwinsu",
+ "kud": "'Auhelawa",
+ "kue": "Kuman (Papua New Guinea)",
+ "kuf": "Western Katu",
+ "kug": "Kupa",
+ "kuh": "Kushi",
+ "kui": "Kuikúro-Kalapálo; Kalapalo",
+ "kuj": "Kuria",
+ "kuk": "Kepo'",
+ "kul": "Kulere",
+ "kum": "Kumyk",
+ "kun": "Kunama",
+ "kuo": "Kumukio",
+ "kup": "Kunimaipa",
+ "kuq": "Karipuna",
+ "kus": "Kusaal",
+ "kut": "Kutenai",
+ "kuu": "Upper Kuskokwim",
+ "kuv": "Kur",
+ "kuw": "Kpagua",
+ "kux": "Kukatja",
+ "kuy": "Kuuku-Ya'u",
+ "kuz": "Kunza",
+ "kv": "Komi",
+ "kva": "Bagvalal",
+ "kvb": "Kubu",
+ "kvc": "Kove",
+ "kvd": "Kui (Indonesia)",
+ "kve": "Kalabakan",
+ "kvf": "Kabalai",
+ "kvg": "Kuni-Boazi",
+ "kvh": "Komodo",
+ "kvi": "Kwang",
+ "kvj": "Psikye",
+ "kvk": "Korean Sign Language",
+ "kvl": "Kayaw",
+ "kvm": "Kendem",
+ "kvn": "Border Kuna",
+ "kvo": "Dobel",
+ "kvp": "Kompane",
+ "kvq": "Geba Karen",
+ "kvr": "Kerinci",
+ "kvt": "Lahta Karen; Lahta",
+ "kvu": "Yinbaw Karen",
+ "kvv": "Kola",
+ "kvw": "Wersing",
+ "kvx": "Parkari Koli",
+ "kvy": "Yintale Karen; Yintale",
+ "kvz": "Tsakwambo; Tsaukambo",
+ "kw": "Cornish",
+ "kwa": "Dâw",
+ "kwb": "Kwa",
+ "kwc": "Likwala",
+ "kwd": "Kwaio",
+ "kwe": "Kwerba",
+ "kwf": "Kwara'ae",
+ "kwg": "Sara Kaba Deme",
+ "kwh": "Kowiai",
+ "kwi": "Awa-Cuaiquer",
+ "kwj": "Kwanga",
+ "kwk": "Kwakiutl",
+ "kwl": "Kofyar",
+ "kwm": "Kwambi",
+ "kwn": "Kwangali",
+ "kwo": "Kwomtari",
+ "kwp": "Kodia",
+ "kwr": "Kwer",
+ "kws": "Kwese",
+ "kwt": "Kwesten",
+ "kwu": "Kwakum",
+ "kwv": "Sara Kaba Náà",
+ "kww": "Kwinti",
+ "kwx": "Khirwar",
+ "kwy": "San Salvador Kongo",
+ "kwz": "Kwadi",
+ "kxa": "Kairiru",
+ "kxb": "Krobu",
+ "kxc": "Konso; Khonso",
+ "kxd": "Brunei",
+ "kxf": "Manumanaw Karen; Manumanaw",
+ "kxh": "Karo (Ethiopia)",
+ "kxi": "Keningau Murut",
+ "kxj": "Kulfa",
+ "kxk": "Zayein Karen",
+ "kxm": "Northern Khmer",
+ "kxn": "Kanowit-Tanjong Melanau",
+ "kxo": "Kanoé",
+ "kxp": "Wadiyara Koli",
+ "kxq": "Smärky Kanum",
+ "kxr": "Koro (Papua New Guinea)",
+ "kxs": "Kangjia",
+ "kxt": "Koiwat",
+ "kxv": "Kuvi",
+ "kxw": "Konai",
+ "kxx": "Likuba",
+ "kxy": "Kayong",
+ "kxz": "Kerewo",
+ "ky": "Kirghiz; Kyrgyz",
+ "kya": "Kwaya",
+ "kyb": "Butbut Kalinga",
+ "kyc": "Kyaka",
+ "kyd": "Karey",
+ "kye": "Krache",
+ "kyf": "Kouya",
+ "kyg": "Keyagana",
+ "kyh": "Karok",
+ "kyi": "Kiput",
+ "kyj": "Karao",
+ "kyk": "Kamayo",
+ "kyl": "Kalapuya",
+ "kym": "Kpatili",
+ "kyn": "Northern Binukidnon",
+ "kyo": "Kelon",
+ "kyp": "Kang",
+ "kyq": "Kenga",
+ "kyr": "Kuruáya",
+ "kys": "Baram Kayan",
+ "kyt": "Kayagar",
+ "kyu": "Western Kayah",
+ "kyv": "Kayort",
+ "kyw": "Kudmali",
+ "kyx": "Rapoisi",
+ "kyy": "Kambaira",
+ "kyz": "Kayabí",
+ "kza": "Western Karaboro",
+ "kzb": "Kaibobo",
+ "kzc": "Bondoukou Kulango",
+ "kzd": "Kadai",
+ "kze": "Kosena",
+ "kzf": "Da'a Kaili",
+ "kzg": "Kikai",
+ "kzi": "Kelabit",
+ "kzk": "Kazukuru",
+ "kzl": "Kayeli",
+ "kzm": "Kais",
+ "kzn": "Kokola",
+ "kzo": "Kaningi",
+ "kzp": "Kaidipang",
+ "kzq": "Kaike",
+ "kzr": "Karang",
+ "kzs": "Sugut Dusun",
+ "kzu": "Kayupulau",
+ "kzv": "Komyandaret",
+ "kzw": "Karirí-Xocó",
+ "kzx": "Kamarian",
+ "kzy": "Kango (Tshopo District)",
+ "kzz": "Kalabra",
+ "la": "Latin",
+ "laa": "Southern Subanen",
+ "lab": "Linear A",
+ "lac": "Lacandon",
+ "lad": "Ladino",
+ "lae": "Pattani",
+ "laf": "Lafofa",
+ "lag": "Langi",
+ "lah": "Lahnda",
+ "lai": "Lambya",
+ "laj": "Lango (Uganda)",
+ "lal": "Lalia",
+ "lam": "Lamba",
+ "lan": "Laru",
+ "lap": "Laka (Chad)",
+ "laq": "Qabiao",
+ "lar": "Larteh",
+ "las": "Lama (Togo)",
+ "lau": "Laba",
+ "law": "Lauje",
+ "lax": "Tiwa",
+ "lay": "Lama Bai",
+ "laz": "Aribwatsa",
+ "lb": "Luxembourgish; Letzeburgesch",
+ "lbb": "Label",
+ "lbc": "Lakkia",
+ "lbe": "Lak",
+ "lbf": "Tinani",
+ "lbg": "Laopang",
+ "lbi": "La'bi",
+ "lbj": "Ladakhi",
+ "lbk": "Central Bontok",
+ "lbl": "Libon Bikol",
+ "lbm": "Lodhi",
+ "lbn": "Rmeet",
+ "lbo": "Laven",
+ "lbq": "Wampar",
+ "lbr": "Lohorung",
+ "lbs": "Libyan Sign Language",
+ "lbt": "Lachi",
+ "lbu": "Labu",
+ "lbv": "Lavatbura-Lamusong",
+ "lbw": "Tolaki",
+ "lbx": "Lawangan",
+ "lby": "Lamalama; Lamu-Lamu",
+ "lbz": "Lardil",
+ "lcc": "Legenyem",
+ "lcd": "Lola",
+ "lce": "Loncong; Sekak",
+ "lcf": "Lubu",
+ "lch": "Luchazi",
+ "lcl": "Lisela",
+ "lcm": "Tungag",
+ "lcp": "Western Lawa",
+ "lcq": "Luhu",
+ "lcs": "Lisabata-Nuniali",
+ "lda": "Kla-Dan",
+ "ldb": "Dũya",
+ "ldd": "Luri",
+ "ldg": "Lenyima",
+ "ldh": "Lamja-Dengsa-Tola",
+ "ldi": "Laari",
+ "ldj": "Lemoro",
+ "ldk": "Leelau",
+ "ldl": "Kaan",
+ "ldm": "Landoma",
+ "ldn": "Láadan",
+ "ldo": "Loo",
+ "ldp": "Tso",
+ "ldq": "Lufu",
+ "lea": "Lega-Shabunda",
+ "leb": "Lala-Bisa",
+ "lec": "Leco",
+ "led": "Lendu",
+ "lee": "Lyélé",
+ "lef": "Lelemi",
+ "leh": "Lenje",
+ "lei": "Lemio",
+ "lej": "Lengola",
+ "lek": "Leipon",
+ "lel": "Lele (Democratic Republic of Congo)",
+ "lem": "Nomaande",
+ "len": "Lenca",
+ "leo": "Leti (Cameroon)",
+ "lep": "Lepcha",
+ "leq": "Lembena",
+ "ler": "Lenkau",
+ "les": "Lese",
+ "let": "Lesing-Gelimi; Amio-Gelimi",
+ "leu": "Kara (Papua New Guinea)",
+ "lev": "Lamma",
+ "lew": "Ledo Kaili",
+ "lex": "Luang",
+ "ley": "Lemolang",
+ "lez": "Lezghian",
+ "lfa": "Lefa",
+ "lfn": "Lingua Franca Nova",
+ "lg": "Ganda; Luganda",
+ "lga": "Lungga",
+ "lgb": "Laghu",
+ "lgg": "Lugbara",
+ "lgh": "Laghuu",
+ "lgi": "Lengilu",
+ "lgk": "Lingarak; Neverver",
+ "lgl": "Wala",
+ "lgm": "Lega-Mwenga",
+ "lgn": "T'apo; Opuuo",
+ "lgo": "Lango (South Sudan)",
+ "lgq": "Logba",
+ "lgr": "Lengo",
+ "lgt": "Pahi",
+ "lgu": "Longgu",
+ "lgz": "Ligenza",
+ "lha": "Laha (Viet Nam)",
+ "lhh": "Laha (Indonesia)",
+ "lhi": "Lahu Shi",
+ "lhl": "Lahul Lohar",
+ "lhm": "Lhomi",
+ "lhn": "Lahanan",
+ "lhp": "Lhokpu",
+ "lhs": "Mlahsö",
+ "lht": "Lo-Toga",
+ "lhu": "Lahu",
+ "li": "Limburgan; Limburger; Limburgish",
+ "lia": "West-Central Limba",
+ "lib": "Likum",
+ "lic": "Hlai",
+ "lid": "Nyindrou",
+ "lie": "Likila",
+ "lif": "Limbu",
+ "lig": "Ligbi",
+ "lih": "Lihir",
+ "lij": "Ligurian",
+ "lik": "Lika",
+ "lil": "Lillooet",
+ "lio": "Liki",
+ "lip": "Sekpele",
+ "liq": "Libido",
+ "lir": "Liberian English",
+ "lis": "Lisu",
+ "liu": "Logorik",
+ "liv": "Liv",
+ "liw": "Col",
+ "lix": "Liabuku",
+ "liy": "Banda-Bambari",
+ "liz": "Libinza",
+ "lja": "Golpa",
+ "lje": "Rampi",
+ "lji": "Laiyolo",
+ "ljl": "Li'o",
+ "ljp": "Lampung Api",
+ "ljw": "Yirandali",
+ "ljx": "Yuru",
+ "lka": "Lakalei",
+ "lkb": "Kabras; Lukabaras",
+ "lkc": "Kucong",
+ "lkd": "Lakondê",
+ "lke": "Kenyi",
+ "lkh": "Lakha",
+ "lki": "Laki",
+ "lkj": "Remun",
+ "lkl": "Laeko-Libuat",
+ "lkm": "Kalaamaya",
+ "lkn": "Lakon; Vure",
+ "lko": "Khayo; Olukhayo",
+ "lkr": "Päri",
+ "lks": "Kisa; Olushisa",
+ "lkt": "Lakota",
+ "lku": "Kungkari",
+ "lky": "Lokoya",
+ "lla": "Lala-Roba",
+ "llb": "Lolo",
+ "llc": "Lele (Guinea)",
+ "lld": "Ladin",
+ "lle": "Lele (Papua New Guinea)",
+ "llf": "Hermit",
+ "llg": "Lole",
+ "llh": "Lamu",
+ "lli": "Teke-Laali",
+ "llj": "Ladji Ladji",
+ "llk": "Lelak",
+ "lll": "Lilau",
+ "llm": "Lasalimu",
+ "lln": "Lele (Chad)",
+ "llp": "North Efate",
+ "llq": "Lolak",
+ "lls": "Lithuanian Sign Language",
+ "llu": "Lau",
+ "llx": "Lauan",
+ "lma": "East Limba",
+ "lmb": "Merei",
+ "lmc": "Limilngan",
+ "lmd": "Lumun",
+ "lme": "Pévé",
+ "lmf": "South Lembata",
+ "lmg": "Lamogai",
+ "lmh": "Lambichhong",
+ "lmi": "Lombi",
+ "lmj": "West Lembata",
+ "lmk": "Lamkang",
+ "lml": "Hano",
+ "lmn": "Lambadi",
+ "lmo": "Lombard",
+ "lmp": "Limbum",
+ "lmq": "Lamatuka",
+ "lmr": "Lamalera",
+ "lmu": "Lamenu",
+ "lmv": "Lomaiviti",
+ "lmw": "Lake Miwok",
+ "lmx": "Laimbue",
+ "lmy": "Lamboya",
+ "ln": "Lingala",
+ "lna": "Langbashe",
+ "lnb": "Mbalanhu",
+ "lnd": "Lundayeh; Lun Bawang",
+ "lng": "Langobardic",
+ "lnh": "Lanoh",
+ "lni": "Daantanai'",
+ "lnj": "Leningitij",
+ "lnl": "South Central Banda",
+ "lnm": "Langam",
+ "lnn": "Lorediakarkar",
+ "lns": "Lamnso'",
+ "lnu": "Longuda",
+ "lnw": "Lanima",
+ "lnz": "Lonzo",
+ "lo": "Lao",
+ "loa": "Loloda",
+ "lob": "Lobi",
+ "loc": "Inonhan",
+ "loe": "Saluan",
+ "lof": "Logol",
+ "log": "Logo",
+ "loh": "Narim",
+ "loi": "Loma (Côte d'Ivoire)",
+ "loj": "Lou",
+ "lok": "Loko",
+ "lol": "Mongo",
+ "lom": "Loma (Liberia)",
+ "lon": "Malawi Lomwe",
+ "loo": "Lombo",
+ "lop": "Lopa",
+ "loq": "Lobala",
+ "lor": "Téén",
+ "los": "Loniu",
+ "lot": "Otuho",
+ "lou": "Louisiana Creole",
+ "lov": "Lopi",
+ "low": "Tampias Lobu",
+ "lox": "Loun",
+ "loy": "Loke",
+ "loz": "Lozi",
+ "lpa": "Lelepa",
+ "lpe": "Lepki",
+ "lpn": "Long Phuri Naga",
+ "lpo": "Lipo",
+ "lpx": "Lopit",
+ "lqr": "Logir",
+ "lra": "Rara Bakati'",
+ "lrc": "Northern Luri",
+ "lre": "Laurentian",
+ "lrg": "Laragia",
+ "lri": "Marachi; Olumarachi",
+ "lrk": "Loarki",
+ "lrl": "Lari",
+ "lrm": "Marama; Olumarama",
+ "lrn": "Lorang",
+ "lro": "Laro",
+ "lrr": "Southern Yamphu",
+ "lrt": "Larantuka Malay",
+ "lrv": "Larevat",
+ "lrz": "Lemerig",
+ "lsa": "Lasgerdi",
+ "lsb": "Burundian Sign Language; Langue des Signes Burundaise",
+ "lsc": "Albarradas Sign Language; Lengua de señas Albarradas",
+ "lsd": "Lishana Deni",
+ "lse": "Lusengo",
+ "lsh": "Lish",
+ "lsi": "Lashi",
+ "lsl": "Latvian Sign Language",
+ "lsm": "Saamia; Olusamia",
+ "lsn": "Tibetan Sign Language",
+ "lso": "Laos Sign Language",
+ "lsp": "Panamanian Sign Language; Lengua de Señas Panameñas",
+ "lsr": "Aruop",
+ "lss": "Lasi",
+ "lst": "Trinidad and Tobago Sign Language",
+ "lsv": "Sivia Sign Language",
+ "lsw": "Seychelles Sign Language; Lalang Siny Seselwa; Langue des Signes Seychelloise",
+ "lsy": "Mauritian Sign Language",
+ "lt": "Lithuanian",
+ "ltc": "Late Middle Chinese",
+ "ltg": "Latgalian",
+ "lth": "Thur",
+ "lti": "Leti (Indonesia)",
+ "ltn": "Latundê",
+ "lto": "Tsotso; Olutsotso",
+ "lts": "Tachoni; Lutachoni",
+ "ltu": "Latu",
+ "lu": "Luba-Katanga",
+ "lua": "Luba-Lulua",
+ "luc": "Aringa",
+ "lud": "Ludian",
+ "lue": "Luvale",
+ "luf": "Laua",
+ "lui": "Luiseno",
+ "luj": "Luna",
+ "luk": "Lunanakha",
+ "lul": "Olu'bo",
+ "lum": "Luimbi",
+ "lun": "Lunda",
+ "luo": "Luo (Kenya and Tanzania); Dholuo",
+ "lup": "Lumbu",
+ "luq": "Lucumi",
+ "lur": "Laura",
+ "lus": "Lushai",
+ "lut": "Lushootseed",
+ "luu": "Lumba-Yakkha",
+ "luv": "Luwati",
+ "luw": "Luo (Cameroon)",
+ "luy": "Luyia; Oluluyia",
+ "luz": "Southern Luri",
+ "lv": "Latvian",
+ "lva": "Maku'a",
+ "lvi": "Lavi",
+ "lvk": "Lavukaleve",
+ "lvs": "Standard Latvian",
+ "lvu": "Levuka",
+ "lwa": "Lwalu",
+ "lwe": "Lewo Eleng",
+ "lwg": "Wanga; Oluwanga",
+ "lwh": "White Lachi",
+ "lwl": "Eastern Lawa",
+ "lwm": "Laomian",
+ "lwo": "Luwo",
+ "lws": "Malawian Sign Language",
+ "lwt": "Lewotobi",
+ "lwu": "Lawu",
+ "lww": "Lewo",
+ "lxm": "Lakurumau",
+ "lya": "Layakha",
+ "lyg": "Lyngngam",
+ "lyn": "Luyana",
+ "lzh": "Literary Chinese",
+ "lzl": "Litzlitz",
+ "lzn": "Leinong Naga",
+ "lzz": "Laz",
+ "maa": "San Jerónimo Tecóatl Mazatec",
+ "mab": "Yutanduchi Mixtec",
+ "mad": "Madurese",
+ "mae": "Bo-Rukul",
+ "maf": "Mafa",
+ "mag": "Magahi",
+ "mai": "Maithili",
+ "maj": "Jalapa De Díaz Mazatec",
+ "mak": "Makasar",
+ "mam": "Mam",
+ "man": "Mandingo; Manding",
+ "map": "Austronesian languages",
+ "maq": "Chiquihuitlán Mazatec",
+ "mas": "Masai",
+ "mat": "San Francisco Matlatzinca",
+ "mau": "Huautla Mazatec",
+ "mav": "Sateré-Mawé",
+ "maw": "Mampruli",
+ "max": "North Moluccan Malay",
+ "maz": "Central Mazahua",
+ "mba": "Higaonon",
+ "mbb": "Western Bukidnon Manobo",
+ "mbc": "Macushi",
+ "mbd": "Dibabawon Manobo",
+ "mbe": "Molale",
+ "mbf": "Baba Malay",
+ "mbh": "Mangseng",
+ "mbi": "Ilianen Manobo",
+ "mbj": "Nadëb",
+ "mbk": "Malol",
+ "mbl": "Maxakalí",
+ "mbm": "Ombamba",
+ "mbn": "Macaguán",
+ "mbo": "Mbo (Cameroon)",
+ "mbp": "Malayo",
+ "mbq": "Maisin",
+ "mbr": "Nukak Makú",
+ "mbs": "Sarangani Manobo",
+ "mbt": "Matigsalug Manobo",
+ "mbu": "Mbula-Bwazza",
+ "mbv": "Mbulungish",
+ "mbw": "Maring",
+ "mbx": "Mari (East Sepik Province)",
+ "mby": "Memoni",
+ "mbz": "Amoltepec Mixtec",
+ "mca": "Maca",
+ "mcb": "Machiguenga",
+ "mcc": "Bitur",
+ "mcd": "Sharanahua",
+ "mce": "Itundujia Mixtec",
+ "mcf": "Matsés",
+ "mcg": "Mapoyo",
+ "mch": "Maquiritari",
+ "mci": "Mese",
+ "mcj": "Mvanip",
+ "mck": "Mbunda",
+ "mcl": "Macaguaje",
+ "mcm": "Malaccan Creole Portuguese",
+ "mcn": "Masana",
+ "mco": "Coatlán Mixe",
+ "mcp": "Makaa",
+ "mcq": "Ese",
+ "mcr": "Menya",
+ "mcs": "Mambai",
+ "mct": "Mengisa",
+ "mcu": "Cameroon Mambila",
+ "mcv": "Minanibai",
+ "mcw": "Mawa (Chad)",
+ "mcx": "Mpiemo",
+ "mcy": "South Watut",
+ "mcz": "Mawan",
+ "mda": "Mada (Nigeria)",
+ "mdb": "Morigi",
+ "mdc": "Male (Papua New Guinea)",
+ "mdd": "Mbum",
+ "mde": "Maba (Chad)",
+ "mdf": "Moksha",
+ "mdg": "Massalat",
+ "mdh": "Maguindanaon",
+ "mdi": "Mamvu",
+ "mdj": "Mangbetu",
+ "mdk": "Mangbutu",
+ "mdl": "Maltese Sign Language",
+ "mdm": "Mayogo",
+ "mdn": "Mbati",
+ "mdp": "Mbala",
+ "mdq": "Mbole",
+ "mdr": "Mandar",
+ "mds": "Maria (Papua New Guinea)",
+ "mdt": "Mbere",
+ "mdu": "Mboko",
+ "mdv": "Santa Lucía Monteverde Mixtec",
+ "mdw": "Mbosi",
+ "mdx": "Dizin",
+ "mdy": "Male (Ethiopia)",
+ "mdz": "Suruí Do Pará",
+ "mea": "Menka",
+ "meb": "Ikobi",
+ "mec": "Marra",
+ "med": "Melpa",
+ "mee": "Mengen",
+ "mef": "Megam",
+ "meh": "Southwestern Tlaxiaco Mixtec",
+ "mei": "Midob",
+ "mej": "Meyah",
+ "mek": "Mekeo",
+ "mel": "Central Melanau",
+ "mem": "Mangala",
+ "men": "Mende (Sierra Leone)",
+ "meo": "Kedah Malay",
+ "mep": "Miriwoong",
+ "meq": "Merey",
+ "mer": "Meru",
+ "mes": "Masmaje",
+ "met": "Mato",
+ "meu": "Motu",
+ "mev": "Mano",
+ "mew": "Maaka",
+ "mey": "Hassaniyya",
+ "mez": "Menominee",
+ "mfa": "Pattani Malay",
+ "mfb": "Bangka",
+ "mfc": "Mba",
+ "mfd": "Mendankwe-Nkwen",
+ "mfe": "Morisyen",
+ "mff": "Naki",
+ "mfg": "Mogofin",
+ "mfh": "Matal",
+ "mfi": "Wandala",
+ "mfj": "Mefele",
+ "mfk": "North Mofu",
+ "mfl": "Putai",
+ "mfm": "Marghi South",
+ "mfn": "Cross River Mbembe",
+ "mfo": "Mbe",
+ "mfp": "Makassar Malay",
+ "mfq": "Moba",
+ "mfr": "Marrithiyel",
+ "mfs": "Mexican Sign Language",
+ "mft": "Mokerang",
+ "mfu": "Mbwela",
+ "mfv": "Mandjak",
+ "mfw": "Mulaha",
+ "mfx": "Melo",
+ "mfy": "Mayo",
+ "mfz": "Mabaan",
+ "mg": "Malagasy",
+ "mga": "Middle Irish (900-1200)",
+ "mgb": "Mararit",
+ "mgc": "Morokodo",
+ "mgd": "Moru",
+ "mge": "Mango",
+ "mgf": "Maklew",
+ "mgg": "Mpumpong",
+ "mgh": "Makhuwa-Meetto",
+ "mgi": "Lijili",
+ "mgj": "Abureni",
+ "mgk": "Mawes",
+ "mgl": "Maleu-Kilenge",
+ "mgm": "Mambae",
+ "mgn": "Mbangi",
+ "mgo": "Meta'",
+ "mgp": "Eastern Magar",
+ "mgq": "Malila",
+ "mgr": "Mambwe-Lungu",
+ "mgs": "Manda (Tanzania)",
+ "mgt": "Mongol",
+ "mgu": "Mailu",
+ "mgv": "Matengo",
+ "mgw": "Matumbi",
+ "mgy": "Mbunga",
+ "mgz": "Mbugwe",
+ "mh": "Marshallese",
+ "mha": "Manda (India)",
+ "mhb": "Mahongwe",
+ "mhc": "Mocho",
+ "mhd": "Mbugu",
+ "mhe": "Besisi; Mah Meri",
+ "mhf": "Mamaa",
+ "mhg": "Margu",
+ "mhi": "Ma'di",
+ "mhj": "Mogholi",
+ "mhk": "Mungaka",
+ "mhl": "Mauwake",
+ "mhm": "Makhuwa-Moniga",
+ "mhn": "Mócheno",
+ "mho": "Mashi (Zambia)",
+ "mhp": "Balinese Malay",
+ "mhq": "Mandan",
+ "mhr": "Eastern Mari",
+ "mhs": "Buru (Indonesia)",
+ "mht": "Mandahuaca",
+ "mhu": "Digaro-Mishmi; Darang Deng",
+ "mhw": "Mbukushu",
+ "mhx": "Maru; Lhaovo",
+ "mhy": "Ma'anyan",
+ "mhz": "Mor (Mor Islands)",
+ "mi": "Maori",
+ "mia": "Miami",
+ "mib": "Atatláhuca Mixtec",
+ "mic": "Mi'kmaq; Micmac",
+ "mid": "Mandaic",
+ "mie": "Ocotepec Mixtec",
+ "mif": "Mofu-Gudur",
+ "mig": "San Miguel El Grande Mixtec",
+ "mih": "Chayuco Mixtec",
+ "mii": "Chigmecatitlán Mixtec",
+ "mij": "Abar; Mungbam",
+ "mik": "Mikasuki",
+ "mil": "Peñoles Mixtec",
+ "mim": "Alacatlatzala Mixtec",
+ "min": "Minangkabau",
+ "mio": "Pinotepa Nacional Mixtec",
+ "mip": "Apasco-Apoala Mixtec",
+ "miq": "Mískito",
+ "mir": "Isthmus Mixe",
+ "mit": "Southern Puebla Mixtec",
+ "miu": "Cacaloxtepec Mixtec",
+ "miw": "Akoye",
+ "mix": "Mixtepec Mixtec",
+ "miy": "Ayutla Mixtec",
+ "miz": "Coatzospan Mixtec",
+ "mjb": "Makalero",
+ "mjc": "San Juan Colorado Mixtec",
+ "mjd": "Northwest Maidu",
+ "mje": "Muskum",
+ "mjg": "Tu",
+ "mjh": "Mwera (Nyasa)",
+ "mji": "Kim Mun",
+ "mjj": "Mawak",
+ "mjk": "Matukar",
+ "mjl": "Mandeali",
+ "mjm": "Medebur",
+ "mjn": "Ma (Papua New Guinea)",
+ "mjo": "Malankuravan",
+ "mjp": "Malapandaram",
+ "mjq": "Malaryan",
+ "mjr": "Malavedan",
+ "mjs": "Miship",
+ "mjt": "Sauria Paharia",
+ "mju": "Manna-Dora",
+ "mjv": "Mannan",
+ "mjw": "Karbi",
+ "mjx": "Mahali",
+ "mjy": "Mahican",
+ "mjz": "Majhi",
+ "mk": "Macedonian",
+ "mka": "Mbre",
+ "mkb": "Mal Paharia",
+ "mkc": "Siliput",
+ "mke": "Mawchi",
+ "mkf": "Miya",
+ "mkg": "Mak (China)",
+ "mkh": "Mon-Khmer languages",
+ "mki": "Dhatki",
+ "mkj": "Mokilese",
+ "mkk": "Byep",
+ "mkl": "Mokole",
+ "mkm": "Moklen",
+ "mkn": "Kupang Malay",
+ "mko": "Mingang Doso",
+ "mkp": "Moikodi",
+ "mkq": "Bay Miwok",
+ "mkr": "Malas",
+ "mks": "Silacayoapan Mixtec",
+ "mkt": "Vamale",
+ "mku": "Konyanka Maninka",
+ "mkv": "Mafea",
+ "mkw": "Kituba (Congo)",
+ "mkx": "Kinamiging Manobo",
+ "mky": "East Makian",
+ "mkz": "Makasae",
+ "ml": "Malayalam",
+ "mla": "Malo",
+ "mlb": "Mbule",
+ "mlc": "Cao Lan",
+ "mle": "Manambu",
+ "mlf": "Mal",
+ "mlh": "Mape",
+ "mli": "Malimpung",
+ "mlj": "Miltu",
+ "mlk": "Ilwana; Kiwilwana",
+ "mll": "Malua Bay",
+ "mlm": "Mulam",
+ "mln": "Malango",
+ "mlo": "Mlomp",
+ "mlp": "Bargam",
+ "mlq": "Western Maninkakan",
+ "mlr": "Vame",
+ "mls": "Masalit",
+ "mlu": "To'abaita",
+ "mlv": "Motlav; Mwotlap",
+ "mlw": "Moloko",
+ "mlx": "Malfaxal; Naha'ai",
+ "mlz": "Malaynon",
+ "mma": "Mama",
+ "mmb": "Momina",
+ "mmc": "Michoacán Mazahua",
+ "mmd": "Maonan",
+ "mme": "Mae",
+ "mmf": "Mundat",
+ "mmg": "North Ambrym",
+ "mmh": "Mehináku",
+ "mmi": "Musar",
+ "mmj": "Majhwar",
+ "mmk": "Mukha-Dora",
+ "mml": "Man Met",
+ "mmm": "Maii",
+ "mmn": "Mamanwa",
+ "mmo": "Mangga Buang",
+ "mmp": "Siawi",
+ "mmq": "Musak",
+ "mmr": "Western Xiangxi Miao",
+ "mmt": "Malalamai",
+ "mmu": "Mmaala",
+ "mmv": "Miriti",
+ "mmw": "Emae",
+ "mmx": "Madak",
+ "mmy": "Migaama",
+ "mmz": "Mabaale",
+ "mn": "Mongolian",
+ "mna": "Mbula",
+ "mnb": "Muna",
+ "mnc": "Manchu",
+ "mnd": "Mondé",
+ "mne": "Naba",
+ "mnf": "Mundani",
+ "mng": "Eastern Mnong",
+ "mnh": "Mono (Democratic Republic of Congo)",
+ "mni": "Manipuri",
+ "mnj": "Munji",
+ "mnk": "Mandinka",
+ "mnl": "Tiale",
+ "mnm": "Mapena",
+ "mnn": "Southern Mnong",
+ "mno": "Manobo languages",
+ "mnp": "Min Bei Chinese",
+ "mnq": "Minriq",
+ "mnr": "Mono (USA)",
+ "mns": "Mansi",
+ "mnu": "Mer",
+ "mnv": "Rennell-Bellona",
+ "mnw": "Mon",
+ "mnx": "Manikion",
+ "mny": "Manyawa",
+ "mnz": "Moni",
+ "moa": "Mwan",
+ "moc": "Mocoví",
+ "mod": "Mobilian",
+ "moe": "Innu; Montagnais",
+ "mog": "Mongondow",
+ "moh": "Mohawk",
+ "moi": "Mboi",
+ "moj": "Monzombo",
+ "mok": "Morori",
+ "mom": "Mangue",
+ "moo": "Monom",
+ "mop": "Mopán Maya",
+ "moq": "Mor (Bomberai Peninsula)",
+ "mor": "Moro",
+ "mos": "Mossi",
+ "mot": "Barí",
+ "mou": "Mogum",
+ "mov": "Mohave",
+ "mow": "Moi (Congo)",
+ "mox": "Molima",
+ "moy": "Shekkacho",
+ "moz": "Mukulu; Gergiko",
+ "mpa": "Mpoto",
+ "mpb": "Malak Malak; Mullukmulluk",
+ "mpc": "Mangarrayi",
+ "mpd": "Machinere",
+ "mpe": "Majang",
+ "mpg": "Marba",
+ "mph": "Maung",
+ "mpi": "Mpade",
+ "mpj": "Martu Wangka; Wangkajunga",
+ "mpk": "Mbara (Chad)",
+ "mpl": "Middle Watut",
+ "mpm": "Yosondúa Mixtec",
+ "mpn": "Mindiri",
+ "mpo": "Miu",
+ "mpp": "Migabac",
+ "mpq": "Matís",
+ "mpr": "Vangunu",
+ "mps": "Dadibi",
+ "mpt": "Mian",
+ "mpu": "Makuráp",
+ "mpv": "Mungkip",
+ "mpw": "Mapidian",
+ "mpx": "Misima-Panaeati",
+ "mpy": "Mapia",
+ "mpz": "Mpi",
+ "mqa": "Maba (Indonesia)",
+ "mqb": "Mbuko",
+ "mqc": "Mangole",
+ "mqe": "Matepi",
+ "mqf": "Momuna",
+ "mqg": "Kota Bangun Kutai Malay",
+ "mqh": "Tlazoyaltepec Mixtec",
+ "mqi": "Mariri",
+ "mqj": "Mamasa",
+ "mqk": "Rajah Kabunsuwan Manobo",
+ "mql": "Mbelime",
+ "mqm": "South Marquesan",
+ "mqn": "Moronene",
+ "mqo": "Modole",
+ "mqp": "Manipa",
+ "mqq": "Minokok",
+ "mqr": "Mander",
+ "mqs": "West Makian",
+ "mqt": "Mok",
+ "mqu": "Mandari",
+ "mqv": "Mosimo",
+ "mqw": "Murupi",
+ "mqx": "Mamuju",
+ "mqy": "Manggarai",
+ "mqz": "Pano",
+ "mr": "Marathi",
+ "mra": "Mlabri",
+ "mrb": "Marino",
+ "mrc": "Maricopa",
+ "mrd": "Western Magar",
+ "mre": "Martha's Vineyard Sign Language",
+ "mrf": "Elseng",
+ "mrg": "Mising",
+ "mrh": "Mara Chin",
+ "mrj": "Western Mari",
+ "mrk": "Hmwaveke",
+ "mrl": "Mortlockese",
+ "mrm": "Merlav; Mwerlap",
+ "mrn": "Cheke Holo",
+ "mro": "Mru",
+ "mrp": "Morouas",
+ "mrq": "North Marquesan",
+ "mrr": "Maria (India)",
+ "mrs": "Maragus",
+ "mrt": "Marghi Central",
+ "mru": "Mono (Cameroon)",
+ "mrv": "Mangareva",
+ "mrw": "Maranao",
+ "mrx": "Maremgi; Dineor",
+ "mry": "Mandaya",
+ "mrz": "Marind",
+ "ms": "Malay (macrolanguage)",
+ "msb": "Masbatenyo",
+ "msc": "Sankaran Maninka",
+ "msd": "Yucatec Maya Sign Language",
+ "mse": "Musey",
+ "msf": "Mekwei",
+ "msg": "Moraid",
+ "msh": "Masikoro Malagasy",
+ "msi": "Sabah Malay",
+ "msj": "Ma (Democratic Republic of Congo)",
+ "msk": "Mansaka",
+ "msl": "Molof; Poule",
+ "msm": "Agusan Manobo",
+ "msn": "Vurës",
+ "mso": "Mombum",
+ "msp": "Maritsauá",
+ "msq": "Caac",
+ "msr": "Mongolian Sign Language",
+ "mss": "West Masela",
+ "msu": "Musom",
+ "msv": "Maslam",
+ "msw": "Mansoanka",
+ "msx": "Moresada",
+ "msy": "Aruamu",
+ "msz": "Momare",
+ "mt": "Maltese",
+ "mta": "Cotabato Manobo",
+ "mtb": "Anyin Morofo",
+ "mtc": "Munit",
+ "mtd": "Mualang",
+ "mte": "Mono (Solomon Islands)",
+ "mtf": "Murik (Papua New Guinea)",
+ "mtg": "Una",
+ "mth": "Munggui",
+ "mti": "Maiwa (Papua New Guinea)",
+ "mtj": "Moskona",
+ "mtk": "Mbe'",
+ "mtl": "Montol",
+ "mtm": "Mator",
+ "mtn": "Matagalpa",
+ "mto": "Totontepec Mixe",
+ "mtp": "Wichí Lhamtés Nocten",
+ "mtq": "Muong",
+ "mtr": "Mewari",
+ "mts": "Yora",
+ "mtt": "Mota",
+ "mtu": "Tututepec Mixtec",
+ "mtv": "Asaro'o",
+ "mtw": "Southern Binukidnon",
+ "mtx": "Tidaá Mixtec",
+ "mty": "Nabi",
+ "mua": "Mundang",
+ "mub": "Mubi",
+ "muc": "Ajumbu",
+ "mud": "Mednyj Aleut",
+ "mue": "Media Lengua",
+ "mug": "Musgu",
+ "muh": "Mündü",
+ "mui": "Musi",
+ "muj": "Mabire",
+ "muk": "Mugom",
+ "mum": "Maiwala",
+ "mun": "Munda languages",
+ "muo": "Nyong",
+ "mup": "Malvi",
+ "muq": "Eastern Xiangxi Miao",
+ "mur": "Murle",
+ "mus": "Creek",
+ "mut": "Western Muria",
+ "muu": "Yaaku",
+ "muv": "Muthuvan",
+ "mux": "Bo-Ung",
+ "muy": "Muyang",
+ "muz": "Mursi",
+ "mva": "Manam",
+ "mvb": "Mattole",
+ "mvd": "Mamboru",
+ "mve": "Marwari (Pakistan)",
+ "mvf": "Peripheral Mongolian",
+ "mvg": "Yucuañe Mixtec",
+ "mvh": "Mulgi",
+ "mvi": "Miyako",
+ "mvk": "Mekmek",
+ "mvl": "Mbara (Australia)",
+ "mvn": "Minaveha",
+ "mvo": "Marovo",
+ "mvp": "Duri",
+ "mvq": "Moere",
+ "mvr": "Marau",
+ "mvs": "Massep",
+ "mvt": "Mpotovoro",
+ "mvu": "Marfa",
+ "mvv": "Tagal Murut",
+ "mvw": "Machinga",
+ "mvx": "Meoswar",
+ "mvy": "Indus Kohistani",
+ "mvz": "Mesqan",
+ "mwa": "Mwatebu",
+ "mwb": "Juwal",
+ "mwc": "Are",
+ "mwe": "Mwera (Chimwera)",
+ "mwf": "Murrinh-Patha",
+ "mwg": "Aiklep",
+ "mwh": "Mouk-Aria",
+ "mwi": "Labo; Ninde",
+ "mwk": "Kita Maninkakan",
+ "mwl": "Mirandese",
+ "mwm": "Sar",
+ "mwn": "Nyamwanga",
+ "mwo": "Central Maewo",
+ "mwp": "Kala Lagaw Ya",
+ "mwq": "Mün Chin",
+ "mwr": "Marwari",
+ "mws": "Mwimbi-Muthambi",
+ "mwt": "Moken",
+ "mwu": "Mittu",
+ "mwv": "Mentawai",
+ "mww": "Hmong Daw",
+ "mwz": "Moingi",
+ "mxa": "Northwest Oaxaca Mixtec",
+ "mxb": "Tezoatlán Mixtec",
+ "mxc": "Manyika",
+ "mxd": "Modang",
+ "mxe": "Mele-Fila",
+ "mxf": "Malgbe",
+ "mxg": "Mbangala",
+ "mxh": "Mvuba",
+ "mxi": "Mozarabic",
+ "mxj": "Miju-Mishmi; Geman Deng",
+ "mxk": "Monumbo",
+ "mxl": "Maxi Gbe",
+ "mxm": "Meramera",
+ "mxn": "Moi (Indonesia)",
+ "mxo": "Mbowe",
+ "mxp": "Tlahuitoltepec Mixe",
+ "mxq": "Juquila Mixe",
+ "mxr": "Murik (Malaysia)",
+ "mxs": "Huitepec Mixtec",
+ "mxt": "Jamiltepec Mixtec",
+ "mxu": "Mada (Cameroon)",
+ "mxv": "Metlatónoc Mixtec",
+ "mxw": "Namo",
+ "mxx": "Mahou; Mawukakan",
+ "mxy": "Southeastern Nochixtlán Mixtec",
+ "mxz": "Central Masela",
+ "my": "Burmese",
+ "myb": "Mbay",
+ "myc": "Mayeka",
+ "mye": "Myene",
+ "myf": "Bambassi",
+ "myg": "Manta",
+ "myh": "Makah",
+ "myj": "Mangayat",
+ "myk": "Mamara Senoufo",
+ "myl": "Moma",
+ "mym": "Me'en",
+ "myn": "Mayan languages",
+ "myo": "Anfillo",
+ "myp": "Pirahã",
+ "myr": "Muniche",
+ "mys": "Mesmes",
+ "myu": "Mundurukú",
+ "myv": "Erzya",
+ "myw": "Muyuw",
+ "myx": "Masaaba",
+ "myy": "Macuna",
+ "myz": "Classical Mandaic",
+ "mza": "Santa María Zacatepec Mixtec",
+ "mzb": "Tumzabt",
+ "mzc": "Madagascar Sign Language",
+ "mzd": "Malimba",
+ "mze": "Morawa",
+ "mzg": "Monastic Sign Language",
+ "mzh": "Wichí Lhamtés Güisnay",
+ "mzi": "Ixcatlán Mazatec",
+ "mzj": "Manya",
+ "mzk": "Nigeria Mambila",
+ "mzl": "Mazatlán Mixe",
+ "mzm": "Mumuye",
+ "mzn": "Mazanderani",
+ "mzo": "Matipuhy",
+ "mzp": "Movima",
+ "mzq": "Mori Atas",
+ "mzr": "Marúbo",
+ "mzs": "Macanese",
+ "mzt": "Mintil",
+ "mzu": "Inapang",
+ "mzv": "Manza",
+ "mzw": "Deg",
+ "mzx": "Mawayana",
+ "mzy": "Mozambican Sign Language",
+ "mzz": "Maiadomu",
+ "na": "Nauru",
+ "naa": "Namla",
+ "nab": "Southern Nambikuára",
+ "nac": "Narak",
+ "nae": "Naka'ela",
+ "naf": "Nabak",
+ "nag": "Naga Pidgin",
+ "nah": "Nahuatl languages",
+ "nai": "North American Indian languages",
+ "naj": "Nalu",
+ "nak": "Nakanai",
+ "nal": "Nalik",
+ "nam": "Ngan'gityemerri",
+ "nan": "Min Nan Chinese",
+ "nao": "Naaba",
+ "nap": "Neapolitan",
+ "naq": "Khoekhoe; Nama (Namibia)",
+ "nar": "Iguta",
+ "nas": "Naasioi",
+ "nat": "Ca̱hungwa̱rya̱; Hungworo",
+ "naw": "Nawuri",
+ "nax": "Nakwi",
+ "nay": "Ngarrindjeri",
+ "naz": "Coatepec Nahuatl",
+ "nb": "Norwegian Bokmål",
+ "nba": "Nyemba",
+ "nbb": "Ndoe",
+ "nbc": "Chang Naga",
+ "nbd": "Ngbinda",
+ "nbe": "Konyak Naga",
+ "nbg": "Nagarchal",
+ "nbh": "Ngamo",
+ "nbi": "Mao Naga",
+ "nbj": "Ngarinyman",
+ "nbk": "Nake",
+ "nbm": "Ngbaka Ma'bo",
+ "nbn": "Kuri",
+ "nbo": "Nkukoli",
+ "nbp": "Nnam",
+ "nbq": "Nggem",
+ "nbr": "Numana",
+ "nbs": "Namibian Sign Language",
+ "nbt": "Na",
+ "nbu": "Rongmei Naga",
+ "nbv": "Ngamambo",
+ "nbw": "Southern Ngbandi",
+ "nby": "Ningera",
+ "nca": "Iyo",
+ "ncb": "Central Nicobarese",
+ "ncc": "Ponam",
+ "ncd": "Nachering",
+ "nce": "Yale",
+ "ncf": "Notsi",
+ "ncg": "Nisga'a",
+ "nch": "Central Huasteca Nahuatl",
+ "nci": "Classical Nahuatl",
+ "ncj": "Northern Puebla Nahuatl",
+ "nck": "Na-kara",
+ "ncl": "Michoacán Nahuatl",
+ "ncm": "Nambo",
+ "ncn": "Nauna",
+ "nco": "Sibe",
+ "ncq": "Northern Katang",
+ "ncr": "Ncane",
+ "ncs": "Nicaraguan Sign Language",
+ "nct": "Chothe Naga",
+ "ncu": "Chumburung",
+ "ncx": "Central Puebla Nahuatl",
+ "ncz": "Natchez",
+ "nd": "North Ndebele",
+ "nda": "Ndasa",
+ "ndb": "Kenswei Nsei",
+ "ndc": "Ndau",
+ "ndd": "Nde-Nsele-Nta",
+ "ndf": "Nadruvian",
+ "ndg": "Ndengereko",
+ "ndh": "Ndali",
+ "ndi": "Samba Leko",
+ "ndj": "Ndamba",
+ "ndk": "Ndaka",
+ "ndl": "Ndolo",
+ "ndm": "Ndam",
+ "ndn": "Ngundi",
+ "ndp": "Ndo",
+ "ndq": "Ndombe",
+ "ndr": "Ndoola",
+ "nds": "Low German; Low Saxon",
+ "ndt": "Ndunga",
+ "ndu": "Dugun",
+ "ndv": "Ndut",
+ "ndw": "Ndobo",
+ "ndx": "Nduga",
+ "ndy": "Lutos",
+ "ndz": "Ndogo",
+ "ne": "Nepali (macrolanguage)",
+ "nea": "Eastern Ngad'a",
+ "neb": "Toura (Côte d'Ivoire)",
+ "nec": "Nedebang",
+ "ned": "Nde-Gbite",
+ "nee": "Nêlêmwa-Nixumwak",
+ "nef": "Nefamese",
+ "neg": "Negidal",
+ "neh": "Nyenkha",
+ "nei": "Neo-Hittite",
+ "nej": "Neko",
+ "nek": "Neku",
+ "nem": "Nemi",
+ "nen": "Nengone",
+ "neo": "Ná-Meo",
+ "neq": "North Central Mixe",
+ "ner": "Yahadian",
+ "nes": "Bhoti Kinnauri",
+ "net": "Nete",
+ "neu": "Neo",
+ "nev": "Nyaheun",
+ "new": "Newari; Nepal Bhasa",
+ "nex": "Neme",
+ "ney": "Neyo",
+ "nez": "Nez Perce",
+ "nfa": "Dhao",
+ "nfd": "Ahwai",
+ "nfl": "Ayiwo; Äiwoo",
+ "nfr": "Nafaanra",
+ "nfu": "Mfumte",
+ "ng": "Ndonga",
+ "nga": "Ngbaka",
+ "ngb": "Northern Ngbandi",
+ "ngc": "Ngombe (Democratic Republic of Congo)",
+ "ngd": "Ngando (Central African Republic)",
+ "nge": "Ngemba",
+ "ngf": "Trans-New Guinea languages",
+ "ngg": "Ngbaka Manza",
+ "ngh": "Nǁng",
+ "ngi": "Ngizim",
+ "ngj": "Ngie",
+ "ngk": "Dalabon",
+ "ngl": "Lomwe",
+ "ngm": "Ngatik Men's Creole",
+ "ngn": "Ngwo",
+ "ngp": "Ngulu",
+ "ngq": "Ngurimi; Ngoreme",
+ "ngr": "Engdewu",
+ "ngs": "Gvoko",
+ "ngt": "Kriang; Ngeq",
+ "ngu": "Guerrero Nahuatl",
+ "ngv": "Nagumi",
+ "ngw": "Ngwaba",
+ "ngx": "Nggwahyi",
+ "ngy": "Tibea",
+ "ngz": "Ngungwel",
+ "nha": "Nhanda",
+ "nhb": "Beng",
+ "nhc": "Tabasco Nahuatl",
+ "nhd": "Chiripá; Ava Guaraní",
+ "nhe": "Eastern Huasteca Nahuatl",
+ "nhf": "Nhuwala",
+ "nhg": "Tetelcingo Nahuatl",
+ "nhh": "Nahari",
+ "nhi": "Zacatlán-Ahuacatlán-Tepetzintla Nahuatl",
+ "nhk": "Isthmus-Cosoleacaque Nahuatl",
+ "nhm": "Morelos Nahuatl",
+ "nhn": "Central Nahuatl",
+ "nho": "Takuu",
+ "nhp": "Isthmus-Pajapan Nahuatl",
+ "nhq": "Huaxcaleca Nahuatl",
+ "nhr": "Naro",
+ "nht": "Ometepec Nahuatl",
+ "nhu": "Noone",
+ "nhv": "Temascaltepec Nahuatl",
+ "nhw": "Western Huasteca Nahuatl",
+ "nhx": "Isthmus-Mecayapan Nahuatl",
+ "nhy": "Northern Oaxaca Nahuatl",
+ "nhz": "Santa María La Alta Nahuatl",
+ "nia": "Nias",
+ "nib": "Nakame",
+ "nic": "Niger-Kordofanian languages",
+ "nid": "Ngandi",
+ "nie": "Niellim",
+ "nif": "Nek",
+ "nig": "Ngalakgan",
+ "nih": "Nyiha (Tanzania)",
+ "nii": "Nii",
+ "nij": "Ngaju",
+ "nik": "Southern Nicobarese",
+ "nil": "Nila",
+ "nim": "Nilamba",
+ "nin": "Ninzo",
+ "nio": "Nganasan",
+ "niq": "Nandi",
+ "nir": "Nimboran",
+ "nis": "Nimi",
+ "nit": "Southeastern Kolami",
+ "niu": "Niuean",
+ "niv": "Gilyak",
+ "niw": "Nimo",
+ "nix": "Hema",
+ "niy": "Ngiti",
+ "niz": "Ningil",
+ "nja": "Nzanyi",
+ "njb": "Nocte Naga",
+ "njd": "Ndonde Hamba",
+ "njh": "Lotha Naga",
+ "nji": "Gudanji",
+ "njj": "Njen",
+ "njl": "Njalgulgule",
+ "njm": "Angami Naga",
+ "njn": "Liangmai Naga",
+ "njo": "Ao Naga",
+ "njr": "Njerep",
+ "njs": "Nisa",
+ "njt": "Ndyuka-Trio Pidgin",
+ "nju": "Ngadjunmaya",
+ "njx": "Kunyi",
+ "njy": "Njyem",
+ "njz": "Nyishi",
+ "nka": "Nkoya",
+ "nkb": "Khoibu Naga",
+ "nkc": "Nkongho",
+ "nkd": "Koireng",
+ "nke": "Duke",
+ "nkf": "Inpui Naga",
+ "nkg": "Nekgini",
+ "nkh": "Khezha Naga",
+ "nki": "Thangal Naga",
+ "nkj": "Nakai",
+ "nkk": "Nokuku",
+ "nkm": "Namat",
+ "nkn": "Nkangala",
+ "nko": "Nkonya",
+ "nkp": "Niuatoputapu",
+ "nkq": "Nkami",
+ "nkr": "Nukuoro",
+ "nks": "North Asmat",
+ "nkt": "Nyika (Tanzania)",
+ "nku": "Bouna Kulango",
+ "nkv": "Nyika (Malawi and Zambia)",
+ "nkw": "Nkutu",
+ "nkx": "Nkoroo",
+ "nkz": "Nkari",
+ "nl": "Dutch; Flemish",
+ "nla": "Ngombale",
+ "nlc": "Nalca",
+ "nle": "East Nyala",
+ "nlg": "Gela",
+ "nli": "Grangali",
+ "nlj": "Nyali",
+ "nlk": "Ninia Yali",
+ "nll": "Nihali",
+ "nlm": "Mankiyali",
+ "nlo": "Ngul",
+ "nlq": "Lao Naga",
+ "nlu": "Nchumbulu",
+ "nlv": "Orizaba Nahuatl",
+ "nlw": "Walangama",
+ "nlx": "Nahali",
+ "nly": "Nyamal",
+ "nlz": "Nalögo",
+ "nma": "Maram Naga",
+ "nmb": "Big Nambas; V'ënen Taut",
+ "nmc": "Ngam",
+ "nmd": "Ndumu",
+ "nme": "Mzieme Naga",
+ "nmf": "Tangkhul Naga (India)",
+ "nmg": "Kwasio",
+ "nmh": "Monsang Naga",
+ "nmi": "Nyam",
+ "nmj": "Ngombe (Central African Republic)",
+ "nmk": "Namakura",
+ "nml": "Ndemli",
+ "nmm": "Manangba",
+ "nmn": "ǃXóõ",
+ "nmo": "Moyon Naga",
+ "nmp": "Nimanbur",
+ "nmq": "Nambya",
+ "nmr": "Nimbari",
+ "nms": "Letemboi",
+ "nmt": "Namonuito",
+ "nmu": "Northeast Maidu",
+ "nmv": "Ngamini",
+ "nmw": "Nimoa; Rifao",
+ "nmx": "Nama (Papua New Guinea)",
+ "nmy": "Namuyi",
+ "nmz": "Nawdm",
+ "nn": "Norwegian Nynorsk",
+ "nna": "Nyangumarta",
+ "nnb": "Nande",
+ "nnc": "Nancere",
+ "nnd": "West Ambae",
+ "nne": "Ngandyera",
+ "nnf": "Ngaing",
+ "nng": "Maring Naga",
+ "nnh": "Ngiemboon",
+ "nni": "North Nuaulu",
+ "nnj": "Nyangatom",
+ "nnk": "Nankina",
+ "nnl": "Northern Rengma Naga",
+ "nnm": "Namia",
+ "nnn": "Ngete",
+ "nnp": "Wancho Naga",
+ "nnq": "Ngindo",
+ "nnr": "Narungga",
+ "nnt": "Nanticoke",
+ "nnu": "Dwang",
+ "nnv": "Nugunu (Australia)",
+ "nnw": "Southern Nuni",
+ "nny": "Nyangga",
+ "nnz": "Nda'nda'",
+ "no": "Norwegian",
+ "noa": "Woun Meu",
+ "noc": "Nuk",
+ "nod": "Northern Thai",
+ "noe": "Nimadi",
+ "nof": "Nomane",
+ "nog": "Nogai",
+ "noh": "Nomu",
+ "noi": "Noiri",
+ "noj": "Nonuya",
+ "nok": "Nooksack",
+ "nol": "Nomlaki",
+ "nom": "Nocamán",
+ "non": "Old Norse",
+ "nop": "Numanggang",
+ "noq": "Ngongo",
+ "nos": "Eastern Nisu",
+ "not": "Nomatsiguenga",
+ "nou": "Ewage-Notu",
+ "nov": "Novial",
+ "now": "Nyambo",
+ "noy": "Noy",
+ "noz": "Nayi",
+ "npa": "Nar Phu",
+ "npb": "Nupbikha",
+ "npg": "Ponyo-Gongwang Naga",
+ "nph": "Phom Naga",
+ "npi": "Nepali (individual language)",
+ "npl": "Southeastern Puebla Nahuatl",
+ "npn": "Mondropolon",
+ "npo": "Pochuri Naga",
+ "nps": "Nipsan",
+ "npu": "Puimei Naga",
+ "npx": "Noipx",
+ "npy": "Napu",
+ "nqg": "Southern Nago",
+ "nqk": "Kura Ede Nago",
+ "nql": "Ngendelengo",
+ "nqm": "Ndom",
+ "nqn": "Nen",
+ "nqo": "N'Ko; N’Ko",
+ "nqq": "Kyan-Karyaw Naga",
+ "nqt": "Nteng",
+ "nqy": "Akyaung Ari Naga",
+ "nr": "South Ndebele",
+ "nra": "Ngom",
+ "nrb": "Nara",
+ "nrc": "Noric",
+ "nre": "Southern Rengma Naga",
+ "nrf": "Jèrriais; Guernésiais",
+ "nrg": "Narango",
+ "nri": "Chokri Naga",
+ "nrk": "Ngarla",
+ "nrl": "Ngarluma",
+ "nrm": "Narom",
+ "nrn": "Norn",
+ "nrp": "North Picene",
+ "nrr": "Norra; Nora",
+ "nrt": "Northern Kalapuya",
+ "nru": "Narua",
+ "nrx": "Ngurmbur",
+ "nrz": "Lala",
+ "nsa": "Sangtam Naga",
+ "nsb": "Lower Nossob",
+ "nsc": "Nshi",
+ "nsd": "Southern Nisu",
+ "nse": "Nsenga",
+ "nsf": "Northwestern Nisu",
+ "nsg": "Ngasa",
+ "nsh": "Ngoshie",
+ "nsi": "Nigerian Sign Language",
+ "nsk": "Naskapi",
+ "nsl": "Norwegian Sign Language",
+ "nsm": "Sumi Naga",
+ "nsn": "Nehan",
+ "nso": "Pedi; Northern Sotho; Sepedi",
+ "nsp": "Nepalese Sign Language",
+ "nsq": "Northern Sierra Miwok",
+ "nsr": "Maritime Sign Language",
+ "nss": "Nali",
+ "nst": "Tase Naga",
+ "nsu": "Sierra Negra Nahuatl",
+ "nsv": "Southwestern Nisu",
+ "nsw": "Navut",
+ "nsx": "Nsongo",
+ "nsy": "Nasal",
+ "nsz": "Nisenan",
+ "ntd": "Northern Tidung",
+ "nte": "Nathembo",
+ "ntg": "Ngantangarra",
+ "nti": "Natioro",
+ "ntj": "Ngaanyatjarra",
+ "ntk": "Ikoma-Nata-Isenye",
+ "ntm": "Nateni",
+ "nto": "Ntomba",
+ "ntp": "Northern Tepehuan",
+ "ntr": "Delo",
+ "ntu": "Natügu",
+ "ntw": "Nottoway",
+ "ntx": "Tangkhul Naga (Myanmar)",
+ "nty": "Mantsi",
+ "ntz": "Natanzi",
+ "nua": "Yuanga",
+ "nub": "Nubian languages",
+ "nuc": "Nukuini",
+ "nud": "Ngala",
+ "nue": "Ngundu",
+ "nuf": "Nusu",
+ "nug": "Nungali",
+ "nuh": "Ndunda",
+ "nui": "Ngumbi",
+ "nuj": "Nyole",
+ "nuk": "Nuu-chah-nulth; Nuuchahnulth",
+ "nul": "Nusa Laut",
+ "num": "Niuafo'ou",
+ "nun": "Anong",
+ "nuo": "Nguôn",
+ "nup": "Nupe-Nupe-Tako",
+ "nuq": "Nukumanu",
+ "nur": "Nukuria",
+ "nus": "Nuer",
+ "nut": "Nung (Viet Nam)",
+ "nuu": "Ngbundu",
+ "nuv": "Northern Nuni",
+ "nuw": "Nguluwan",
+ "nux": "Mehek",
+ "nuy": "Nunggubuyu",
+ "nuz": "Tlamacazapa Nahuatl",
+ "nv": "Navajo; Navaho",
+ "nvh": "Nasarian",
+ "nvm": "Namiae",
+ "nvo": "Nyokon",
+ "nwa": "Nawathinehena",
+ "nwb": "Nyabwa",
+ "nwc": "Classical Newari; Classical Nepal Bhasa; Old Newari",
+ "nwe": "Ngwe",
+ "nwg": "Ngayawung",
+ "nwi": "Southwest Tanna",
+ "nwm": "Nyamusa-Molo",
+ "nwo": "Nauo",
+ "nwr": "Nawaru",
+ "nww": "Ndwewe",
+ "nwx": "Middle Newar",
+ "nwy": "Nottoway-Meherrin",
+ "nxa": "Nauete",
+ "nxd": "Ngando (Democratic Republic of Congo)",
+ "nxe": "Nage",
+ "nxg": "Ngad'a",
+ "nxi": "Nindi",
+ "nxk": "Koki Naga",
+ "nxl": "South Nuaulu",
+ "nxm": "Numidian",
+ "nxn": "Ngawun",
+ "nxo": "Ndambomo",
+ "nxq": "Naxi",
+ "nxr": "Ninggerum",
+ "nxx": "Nafri",
+ "ny": "Nyanja; Chewa; Chichewa",
+ "nyb": "Nyangbo",
+ "nyc": "Nyanga-li",
+ "nyd": "Nyore; Olunyole",
+ "nye": "Nyengo",
+ "nyf": "Giryama; Kigiryama",
+ "nyg": "Nyindu",
+ "nyh": "Nyikina",
+ "nyi": "Ama (Sudan)",
+ "nyj": "Nyanga",
+ "nyk": "Nyaneka",
+ "nyl": "Nyeu",
+ "nym": "Nyamwezi",
+ "nyn": "Nyankole",
+ "nyo": "Nyoro",
+ "nyp": "Nyang'i",
+ "nyq": "Nayini",
+ "nyr": "Nyiha (Malawi)",
+ "nys": "Nyungar",
+ "nyt": "Nyawaygi",
+ "nyu": "Nyungwe",
+ "nyv": "Nyulnyul",
+ "nyw": "Nyaw",
+ "nyx": "Nganyaywana",
+ "nyy": "Nyakyusa-Ngonde",
+ "nza": "Tigon Mbembe",
+ "nzb": "Njebi",
+ "nzd": "Nzadi",
+ "nzi": "Nzima",
+ "nzk": "Nzakara",
+ "nzm": "Zeme Naga",
+ "nzs": "New Zealand Sign Language",
+ "nzu": "Teke-Nzikou",
+ "nzy": "Nzakambay",
+ "nzz": "Nanga Dama Dogon",
+ "oaa": "Orok",
+ "oac": "Oroch",
+ "oar": "Old Aramaic (up to 700 BCE); Ancient Aramaic (up to 700 BCE)",
+ "oav": "Old Avar",
+ "obi": "Obispeño",
+ "obk": "Southern Bontok",
+ "obl": "Oblo",
+ "obm": "Moabite",
+ "obo": "Obo Manobo",
+ "obr": "Old Burmese",
+ "obt": "Old Breton",
+ "obu": "Obulom",
+ "oc": "Occitan (post 1500)",
+ "oca": "Ocaina",
+ "och": "Old Chinese",
+ "ocm": "Old Cham",
+ "oco": "Old Cornish",
+ "ocu": "Atzingo Matlatzinca",
+ "oda": "Odut",
+ "odk": "Od",
+ "odt": "Old Dutch",
+ "odu": "Odual",
+ "ofo": "Ofo",
+ "ofs": "Old Frisian",
+ "ofu": "Efutop",
+ "ogb": "Ogbia",
+ "ogc": "Ogbah",
+ "oge": "Old Georgian",
+ "ogg": "Ogbogolo",
+ "ogo": "Khana",
+ "ogu": "Ogbronuagum",
+ "oht": "Old Hittite",
+ "ohu": "Old Hungarian",
+ "oia": "Oirata",
+ "oie": "Okolie",
+ "oin": "Inebu One",
+ "oj": "Ojibwa",
+ "ojb": "Northwestern Ojibwa",
+ "ojc": "Central Ojibwa",
+ "ojg": "Eastern Ojibwa",
+ "ojp": "Old Japanese",
+ "ojs": "Severn Ojibwa",
+ "ojv": "Ontong Java",
+ "ojw": "Western Ojibwa",
+ "oka": "Okanagan",
+ "okb": "Okobo",
+ "okc": "Kobo",
+ "okd": "Okodia",
+ "oke": "Okpe (Southwestern Edo)",
+ "okg": "Koko Babangk",
+ "okh": "Koresh-e Rostam",
+ "oki": "Okiek",
+ "okj": "Oko-Juwoi",
+ "okk": "Kwamtim One",
+ "okl": "Old Kentish Sign Language",
+ "okm": "Middle Korean (10th-16th cent.)",
+ "okn": "Oki-No-Erabu",
+ "oko": "Old Korean (3rd-9th cent.)",
+ "okr": "Kirike",
+ "oks": "Oko-Eni-Osayen",
+ "oku": "Oku",
+ "okv": "Orokaiva",
+ "okx": "Okpe (Northwestern Edo)",
+ "okz": "Old Khmer",
+ "ola": "Walungge",
+ "old": "Mochi",
+ "ole": "Olekha",
+ "olk": "Olkol",
+ "olm": "Oloma",
+ "olo": "Livvi",
+ "olr": "Olrat",
+ "olt": "Old Lithuanian",
+ "olu": "Kuvale",
+ "om": "Oromo",
+ "oma": "Omaha-Ponca",
+ "omb": "East Ambae",
+ "omc": "Mochica",
+ "omg": "Omagua",
+ "omi": "Omi",
+ "omk": "Omok",
+ "oml": "Ombo",
+ "omn": "Minoan",
+ "omo": "Utarmbung",
+ "omp": "Old Manipuri",
+ "omq": "Oto-Manguean languages",
+ "omr": "Old Marathi",
+ "omt": "Omotik",
+ "omu": "Omurano",
+ "omv": "Omotic languages",
+ "omw": "South Tairora",
+ "omx": "Old Mon",
+ "omy": "Old Malay",
+ "ona": "Ona",
+ "onb": "Lingao",
+ "one": "Oneida",
+ "ong": "Olo",
+ "oni": "Onin",
+ "onj": "Onjob",
+ "onk": "Kabore One",
+ "onn": "Onobasulu",
+ "ono": "Onondaga",
+ "onp": "Sartang",
+ "onr": "Northern One",
+ "ons": "Ono",
+ "ont": "Ontenu",
+ "onu": "Unua",
+ "onw": "Old Nubian",
+ "onx": "Onin Based Pidgin",
+ "ood": "Tohono O'odham",
+ "oog": "Ong",
+ "oon": "Önge",
+ "oor": "Oorlams",
+ "oos": "Old Ossetic",
+ "opa": "Okpamheri",
+ "opk": "Kopkaka",
+ "opm": "Oksapmin",
+ "opo": "Opao",
+ "opt": "Opata",
+ "opy": "Ofayé",
+ "or": "Oriya (macrolanguage); Odia (macrolanguage)",
+ "ora": "Oroha",
+ "orc": "Orma",
+ "ore": "Orejón",
+ "org": "Oring",
+ "orh": "Oroqen",
+ "orn": "Orang Kanaq",
+ "oro": "Orokolo",
+ "orr": "Oruma",
+ "ors": "Orang Seletar",
+ "ort": "Adivasi Oriya",
+ "oru": "Ormuri",
+ "orv": "Old Russian",
+ "orw": "Oro Win",
+ "orx": "Oro",
+ "ory": "Odia (individual language); Oriya (individual language)",
+ "orz": "Ormu",
+ "os": "Ossetian; Ossetic",
+ "osa": "Osage",
+ "osc": "Oscan",
+ "osi": "Osing",
+ "osn": "Old Sundanese",
+ "oso": "Ososo",
+ "osp": "Old Spanish",
+ "ost": "Osatu",
+ "osu": "Southern One",
+ "osx": "Old Saxon",
+ "ota": "Ottoman Turkish (1500-1928)",
+ "otb": "Old Tibetan",
+ "otd": "Ot Danum",
+ "ote": "Mezquital Otomi",
+ "oti": "Oti",
+ "otk": "Old Turkish",
+ "otl": "Tilapa Otomi",
+ "otm": "Eastern Highland Otomi",
+ "otn": "Tenango Otomi",
+ "oto": "Otomian languages",
+ "otq": "Querétaro Otomi",
+ "otr": "Otoro",
+ "ots": "Estado de México Otomi",
+ "ott": "Temoaya Otomi",
+ "otu": "Otuke",
+ "otw": "Ottawa",
+ "otx": "Texcatepec Otomi",
+ "oty": "Old Tamil",
+ "otz": "Ixtenco Otomi",
+ "oua": "Tagargrent",
+ "oub": "Glio-Oubi",
+ "oue": "Oune",
+ "oui": "Old Uighur",
+ "oum": "Ouma",
+ "ovd": "Elfdalian; Övdalian",
+ "owi": "Owiniga",
+ "owl": "Old Welsh",
+ "oyb": "Oy",
+ "oyd": "Oyda",
+ "oym": "Wayampi",
+ "oyy": "Oya'oya",
+ "ozm": "Koonzime",
+ "pa": "Panjabi; Punjabi",
+ "paa": "Papuan languages",
+ "pab": "Parecís",
+ "pac": "Pacoh",
+ "pad": "Paumarí",
+ "pae": "Pagibete",
+ "paf": "Paranawát",
+ "pag": "Pangasinan",
+ "pah": "Tenharim",
+ "pai": "Pe",
+ "pak": "Parakanã",
+ "pal": "Pahlavi",
+ "pam": "Pampanga; Kapampangan",
+ "pao": "Northern Paiute",
+ "pap": "Papiamento",
+ "paq": "Parya",
+ "par": "Panamint; Timbisha",
+ "pas": "Papasena",
+ "pau": "Palauan",
+ "pav": "Pakaásnovos",
+ "paw": "Pawnee",
+ "pax": "Pankararé",
+ "pay": "Pech",
+ "paz": "Pankararú",
+ "pbb": "Páez",
+ "pbc": "Patamona",
+ "pbe": "Mezontla Popoloca",
+ "pbf": "Coyotepec Popoloca",
+ "pbg": "Paraujano",
+ "pbh": "E'ñapa Woromaipu",
+ "pbi": "Parkwa",
+ "pbl": "Mak (Nigeria)",
+ "pbm": "Puebla Mazatec",
+ "pbn": "Kpasam",
+ "pbo": "Papel",
+ "pbp": "Badyara",
+ "pbr": "Pangwa",
+ "pbs": "Central Pame",
+ "pbt": "Southern Pashto",
+ "pbu": "Northern Pashto",
+ "pbv": "Pnar",
+ "pby": "Pyu (Papua New Guinea)",
+ "pca": "Santa Inés Ahuatempan Popoloca",
+ "pcb": "Pear",
+ "pcc": "Bouyei",
+ "pcd": "Picard",
+ "pce": "Ruching Palaung",
+ "pcf": "Paliyan",
+ "pcg": "Paniya",
+ "pch": "Pardhan",
+ "pci": "Duruwa",
+ "pcj": "Parenga",
+ "pck": "Paite Chin",
+ "pcl": "Pardhi",
+ "pcm": "Nigerian Pidgin",
+ "pcn": "Piti",
+ "pcp": "Pacahuara",
+ "pcw": "Pyapun",
+ "pda": "Anam",
+ "pdc": "Pennsylvania German",
+ "pdi": "Pa Di",
+ "pdn": "Podena; Fedan",
+ "pdo": "Padoe",
+ "pdt": "Plautdietsch",
+ "pdu": "Kayan",
+ "pea": "Peranakan Indonesian",
+ "peb": "Eastern Pomo",
+ "ped": "Mala (Papua New Guinea)",
+ "pee": "Taje",
+ "pef": "Northeastern Pomo",
+ "peg": "Pengo",
+ "peh": "Bonan",
+ "pei": "Chichimeca-Jonaz",
+ "pej": "Northern Pomo",
+ "pek": "Penchal",
+ "pel": "Pekal",
+ "pem": "Phende",
+ "peo": "Old Persian (ca. 600-400 B.C.)",
+ "pep": "Kunja",
+ "peq": "Southern Pomo",
+ "pes": "Iranian Persian",
+ "pev": "Pémono",
+ "pex": "Petats",
+ "pey": "Petjo",
+ "pez": "Eastern Penan",
+ "pfa": "Pááfang",
+ "pfe": "Pere",
+ "pfl": "Pfaelzisch",
+ "pga": "Sudanese Creole Arabic",
+ "pgd": "Gāndhārī",
+ "pgg": "Pangwali",
+ "pgi": "Pagi",
+ "pgk": "Rerep",
+ "pgl": "Primitive Irish",
+ "pgn": "Paelignian",
+ "pgs": "Pangseng",
+ "pgu": "Pagu",
+ "pgz": "Papua New Guinean Sign Language",
+ "pha": "Pa-Hng",
+ "phd": "Phudagi",
+ "phg": "Phuong",
+ "phh": "Phukha",
+ "phi": "Philippine languages",
+ "phj": "Pahari",
+ "phk": "Phake",
+ "phl": "Phalura; Palula",
+ "phm": "Phimbi",
+ "phn": "Phoenician",
+ "pho": "Phunoi",
+ "phq": "Phana'",
+ "phr": "Pahari-Potwari",
+ "pht": "Phu Thai",
+ "phu": "Phuan",
+ "phv": "Pahlavani",
+ "phw": "Phangduwali",
+ "pi": "Pali",
+ "pia": "Pima Bajo",
+ "pib": "Yine",
+ "pic": "Pinji",
+ "pid": "Piaroa",
+ "pie": "Piro",
+ "pif": "Pingelapese",
+ "pig": "Pisabo",
+ "pih": "Pitcairn-Norfolk",
+ "pij": "Pijao",
+ "pil": "Yom",
+ "pim": "Powhatan",
+ "pin": "Piame",
+ "pio": "Piapoco",
+ "pip": "Pero",
+ "pir": "Piratapuyo",
+ "pis": "Pijin",
+ "pit": "Pitta Pitta",
+ "piu": "Pintupi-Luritja",
+ "piv": "Pileni; Vaeakau-Taumako",
+ "piw": "Pimbwe",
+ "pix": "Piu",
+ "piy": "Piya-Kwonci",
+ "piz": "Pije",
+ "pjt": "Pitjantjatjara",
+ "pka": "Ardhamāgadhī Prākrit",
+ "pkb": "Pokomo; Kipfokomo",
+ "pkc": "Paekche",
+ "pkg": "Pak-Tong",
+ "pkh": "Pankhu",
+ "pkn": "Pakanha",
+ "pko": "Pökoot",
+ "pkp": "Pukapuka",
+ "pkr": "Attapady Kurumba",
+ "pks": "Pakistan Sign Language",
+ "pkt": "Maleng",
+ "pku": "Paku",
+ "pl": "Polish",
+ "pla": "Miani",
+ "plb": "Polonombauk",
+ "plc": "Central Palawano",
+ "pld": "Polari",
+ "ple": "Palu'e",
+ "plf": "Central Malayo-Polynesian languages",
+ "plg": "Pilagá",
+ "plh": "Paulohi",
+ "plj": "Polci",
+ "plk": "Kohistani Shina",
+ "pll": "Shwe Palaung",
+ "pln": "Palenquero",
+ "plo": "Oluta Popoluca",
+ "plq": "Palaic",
+ "plr": "Palaka Senoufo",
+ "pls": "San Marcos Tlacoyalco Popoloca; San Marcos Tlalcoyalco Popoloca",
+ "plt": "Plateau Malagasy",
+ "plu": "Palikúr",
+ "plv": "Southwest Palawano",
+ "plw": "Brooke's Point Palawano",
+ "ply": "Bolyu",
+ "plz": "Paluan",
+ "pma": "Paama",
+ "pmb": "Pambia",
+ "pmd": "Pallanganmiddang",
+ "pme": "Pwaamei",
+ "pmf": "Pamona",
+ "pmh": "Māhārāṣṭri Prākrit",
+ "pmi": "Northern Pumi",
+ "pmj": "Southern Pumi",
+ "pmk": "Pamlico",
+ "pml": "Lingua Franca",
+ "pmm": "Pomo",
+ "pmn": "Pam",
+ "pmo": "Pom",
+ "pmq": "Northern Pame",
+ "pmr": "Paynamar",
+ "pms": "Piemontese",
+ "pmt": "Tuamotuan",
+ "pmw": "Plains Miwok",
+ "pmx": "Poumei Naga",
+ "pmy": "Papuan Malay",
+ "pmz": "Southern Pame",
+ "pna": "Punan Bah-Biau",
+ "pnb": "Western Panjabi",
+ "pnc": "Pannei",
+ "pnd": "Mpinda",
+ "pne": "Western Penan",
+ "png": "Pangu; Pongu",
+ "pnh": "Penrhyn",
+ "pni": "Aoheng",
+ "pnj": "Pinjarup",
+ "pnk": "Paunaka",
+ "pnl": "Paleni",
+ "pnm": "Punan Batu 1",
+ "pnn": "Pinai-Hagahai",
+ "pno": "Panobo",
+ "pnp": "Pancana",
+ "pnq": "Pana (Burkina Faso)",
+ "pnr": "Panim",
+ "pns": "Ponosakan",
+ "pnt": "Pontic",
+ "pnu": "Jiongnai Bunu",
+ "pnv": "Pinigura",
+ "pnw": "Banyjima; Panytyima",
+ "pnx": "Phong-Kniang",
+ "pny": "Pinyin",
+ "pnz": "Pana (Central African Republic)",
+ "poc": "Poqomam",
+ "poe": "San Juan Atzingo Popoloca",
+ "pof": "Poke",
+ "pog": "Potiguára",
+ "poh": "Poqomchi'",
+ "poi": "Highland Popoluca",
+ "pok": "Pokangá",
+ "pom": "Southeastern Pomo",
+ "pon": "Pohnpeian",
+ "poo": "Central Pomo",
+ "pop": "Pwapwâ",
+ "poq": "Texistepec Popoluca",
+ "pos": "Sayula Popoluca",
+ "pot": "Potawatomi",
+ "pov": "Upper Guinea Crioulo",
+ "pow": "San Felipe Otlaltepec Popoloca",
+ "pox": "Polabian",
+ "poy": "Pogolo",
+ "poz": "Malayo-Polynesian languages",
+ "ppe": "Papi",
+ "ppi": "Paipai",
+ "ppk": "Uma",
+ "ppl": "Pipil; Nicarao",
+ "ppm": "Papuma",
+ "ppn": "Papapana",
+ "ppo": "Folopa",
+ "ppp": "Pelende",
+ "ppq": "Pei",
+ "pps": "San Luís Temalacayuca Popoloca",
+ "ppt": "Pare",
+ "ppu": "Papora",
+ "pqa": "Pa'a",
+ "pqe": "Eastern Malayo-Polynesian languages",
+ "pqm": "Malecite-Passamaquoddy",
+ "pqw": "Western Malayo-Polynesian languages",
+ "pra": "Prakrit languages",
+ "prc": "Parachi",
+ "prd": "Parsi-Dari",
+ "pre": "Principense",
+ "prf": "Paranan",
+ "prg": "Prussian",
+ "prh": "Porohanon",
+ "pri": "Paicî",
+ "prk": "Parauk",
+ "prl": "Peruvian Sign Language",
+ "prm": "Kibiri",
+ "prn": "Prasuni",
+ "pro": "Old Provençal (to 1500); Old Occitan (to 1500)",
+ "prp": "Parsi",
+ "prq": "Ashéninka Perené",
+ "prr": "Puri",
+ "prs": "Dari; Afghan Persian",
+ "prt": "Phai",
+ "pru": "Puragi",
+ "prw": "Parawen",
+ "prx": "Purik",
+ "prz": "Providencia Sign Language",
+ "ps": "Pushto; Pashto",
+ "psa": "Asue Awyu",
+ "psc": "Iranian Sign Language; Persian Sign Language",
+ "psd": "Plains Indian Sign Language",
+ "pse": "Central Malay",
+ "psg": "Penang Sign Language",
+ "psh": "Southwest Pashai; Southwest Pashayi",
+ "psi": "Southeast Pashai; Southeast Pashayi",
+ "psl": "Puerto Rican Sign Language",
+ "psm": "Pauserna",
+ "psn": "Panasuan",
+ "pso": "Polish Sign Language",
+ "psp": "Philippine Sign Language",
+ "psq": "Pasi",
+ "psr": "Portuguese Sign Language",
+ "pss": "Kaulong",
+ "pst": "Central Pashto",
+ "psu": "Sauraseni Prākrit",
+ "psw": "Port Sandwich",
+ "psy": "Piscataway",
+ "pt": "Portuguese",
+ "pta": "Pai Tavytera",
+ "pth": "Pataxó Hã-Ha-Hãe",
+ "pti": "Pindiini; Wangkatha",
+ "ptn": "Patani",
+ "pto": "Zo'é",
+ "ptp": "Patep",
+ "ptq": "Pattapu",
+ "ptr": "Piamatsina",
+ "ptt": "Enrekang",
+ "ptu": "Bambam",
+ "ptv": "Port Vato",
+ "ptw": "Pentlatch",
+ "pty": "Pathiya",
+ "pua": "Western Highland Purepecha",
+ "pub": "Purum",
+ "puc": "Punan Merap",
+ "pud": "Punan Aput",
+ "pue": "Puelche",
+ "puf": "Punan Merah",
+ "pug": "Phuie",
+ "pui": "Puinave",
+ "puj": "Punan Tubu",
+ "pum": "Puma",
+ "puo": "Puoc",
+ "pup": "Pulabu",
+ "puq": "Puquina",
+ "pur": "Puruborá",
+ "put": "Putoh",
+ "puu": "Punu",
+ "puw": "Puluwatese",
+ "pux": "Puare",
+ "puy": "Purisimeño",
+ "pwa": "Pawaia",
+ "pwb": "Panawa",
+ "pwg": "Gapapaiwa",
+ "pwi": "Patwin",
+ "pwm": "Molbog",
+ "pwn": "Paiwan",
+ "pwo": "Pwo Western Karen",
+ "pwr": "Powari",
+ "pww": "Pwo Northern Karen",
+ "pxm": "Quetzaltepec Mixe",
+ "pye": "Pye Krumen",
+ "pym": "Fyam",
+ "pyn": "Poyanáwa",
+ "pys": "Paraguayan Sign Language; Lengua de Señas del Paraguay",
+ "pyu": "Puyuma",
+ "pyx": "Pyu (Myanmar)",
+ "pyy": "Pyen",
+ "pzh": "Pazeh",
+ "pzn": "Jejara Naga; Para Naga",
+ "qu": "Quechua",
+ "qua": "Quapaw",
+ "qub": "Huallaga Huánuco Quechua",
+ "quc": "K'iche'; Quiché",
+ "qud": "Calderón Highland Quichua",
+ "quf": "Lambayeque Quechua",
+ "qug": "Chimborazo Highland Quichua",
+ "quh": "South Bolivian Quechua",
+ "qui": "Quileute",
+ "quk": "Chachapoyas Quechua",
+ "qul": "North Bolivian Quechua",
+ "qum": "Sipacapense",
+ "qun": "Quinault",
+ "qup": "Southern Pastaza Quechua",
+ "quq": "Quinqui",
+ "qur": "Yanahuanca Pasco Quechua",
+ "qus": "Santiago del Estero Quichua",
+ "quv": "Sacapulteco",
+ "quw": "Tena Lowland Quichua",
+ "qux": "Yauyos Quechua",
+ "quy": "Ayacucho Quechua",
+ "quz": "Cusco Quechua",
+ "qva": "Ambo-Pasco Quechua",
+ "qvc": "Cajamarca Quechua",
+ "qve": "Eastern Apurímac Quechua",
+ "qvh": "Huamalíes-Dos de Mayo Huánuco Quechua",
+ "qvi": "Imbabura Highland Quichua",
+ "qvj": "Loja Highland Quichua",
+ "qvl": "Cajatambo North Lima Quechua",
+ "qvm": "Margos-Yarowilca-Lauricocha Quechua",
+ "qvn": "North Junín Quechua",
+ "qvo": "Napo Lowland Quechua",
+ "qvp": "Pacaraos Quechua",
+ "qvs": "San Martín Quechua",
+ "qvw": "Huaylla Wanca Quechua",
+ "qvy": "Queyu",
+ "qvz": "Northern Pastaza Quichua",
+ "qwa": "Corongo Ancash Quechua",
+ "qwc": "Classical Quechua",
+ "qwe": "Quechuan (family)",
+ "qwh": "Huaylas Ancash Quechua",
+ "qwm": "Kuman (Russia)",
+ "qws": "Sihuas Ancash Quechua",
+ "qwt": "Kwalhioqua-Tlatskanai",
+ "qxa": "Chiquián Ancash Quechua",
+ "qxc": "Chincha Quechua",
+ "qxh": "Panao Huánuco Quechua",
+ "qxl": "Salasaca Highland Quichua",
+ "qxn": "Northern Conchucos Ancash Quechua",
+ "qxo": "Southern Conchucos Ancash Quechua",
+ "qxp": "Puno Quechua",
+ "qxq": "Qashqa'i",
+ "qxr": "Cañar Highland Quichua",
+ "qxs": "Southern Qiang",
+ "qxt": "Santa Ana de Tusi Pasco Quechua",
+ "qxu": "Arequipa-La Unión Quechua",
+ "qxw": "Jauja Wanca Quechua",
+ "qya": "Quenya",
+ "qyp": "Quiripi",
+ "raa": "Dungmali",
+ "rab": "Camling",
+ "rac": "Rasawa",
+ "rad": "Rade",
+ "raf": "Western Meohang",
+ "rag": "Logooli; Lulogooli",
+ "rah": "Rabha",
+ "rai": "Ramoaaina",
+ "raj": "Rajasthani",
+ "rak": "Tulu-Bohuai",
+ "ral": "Ralte",
+ "ram": "Canela",
+ "ran": "Riantana",
+ "rao": "Rao",
+ "rap": "Rapanui",
+ "raq": "Saam",
+ "rar": "Rarotongan; Cook Islands Maori",
+ "ras": "Tegali",
+ "rat": "Razajerdi",
+ "rau": "Raute",
+ "rav": "Sampang",
+ "raw": "Rawang",
+ "rax": "Rang",
+ "ray": "Rapa",
+ "raz": "Rahambuu",
+ "rbb": "Rumai Palaung",
+ "rbk": "Northern Bontok",
+ "rbl": "Miraya Bikol",
+ "rbp": "Barababaraba",
+ "rcf": "Réunion Creole French",
+ "rdb": "Rudbari",
+ "rea": "Rerau",
+ "reb": "Rembong",
+ "ree": "Rejang Kayan",
+ "reg": "Kara (Tanzania)",
+ "rei": "Reli",
+ "rej": "Rejang",
+ "rel": "Rendille",
+ "rem": "Remo",
+ "ren": "Rengao",
+ "rer": "Rer Bare",
+ "res": "Reshe",
+ "ret": "Retta",
+ "rey": "Reyesano",
+ "rga": "Roria",
+ "rge": "Romano-Greek",
+ "rgk": "Rangkas",
+ "rgn": "Romagnol",
+ "rgr": "Resígaro",
+ "rgs": "Southern Roglai",
+ "rgu": "Ringgou",
+ "rhg": "Rohingya",
+ "rhp": "Yahang",
+ "ria": "Riang (India)",
+ "rib": "Bribri Sign Language",
+ "rif": "Tarifit",
+ "ril": "Riang Lang; Riang (Myanmar)",
+ "rim": "Nyaturu",
+ "rin": "Nungu",
+ "rir": "Ribun",
+ "rit": "Ritharrngu",
+ "riu": "Riung",
+ "rjg": "Rajong",
+ "rji": "Raji",
+ "rjs": "Rajbanshi",
+ "rka": "Kraol",
+ "rkb": "Rikbaktsa",
+ "rkh": "Rakahanga-Manihiki",
+ "rki": "Rakhine",
+ "rkm": "Marka",
+ "rkt": "Rangpuri; Kamta",
+ "rkw": "Arakwal",
+ "rm": "Romansh",
+ "rma": "Rama",
+ "rmb": "Rembarrnga",
+ "rmc": "Carpathian Romani",
+ "rmd": "Traveller Danish",
+ "rme": "Angloromani",
+ "rmf": "Kalo Finnish Romani",
+ "rmg": "Traveller Norwegian",
+ "rmh": "Murkim",
+ "rmi": "Lomavren",
+ "rmk": "Romkun",
+ "rml": "Baltic Romani",
+ "rmm": "Roma",
+ "rmn": "Balkan Romani",
+ "rmo": "Sinte Romani",
+ "rmp": "Rempi",
+ "rmq": "Caló",
+ "rms": "Romanian Sign Language",
+ "rmt": "Domari",
+ "rmu": "Tavringer Romani",
+ "rmv": "Romanova",
+ "rmw": "Welsh Romani",
+ "rmx": "Romam",
+ "rmy": "Vlax Romani",
+ "rmz": "Marma",
+ "rn": "Rundi",
+ "rnb": "Brunca Sign Language",
+ "rnd": "Ruund",
+ "rng": "Ronga",
+ "rnl": "Ranglong",
+ "rnn": "Roon",
+ "rnp": "Rongpo",
+ "rnr": "Nari Nari",
+ "rnw": "Rungwa",
+ "ro": "Romanian; Moldavian; Moldovan",
+ "roa": "Romance languages",
+ "rob": "Tae'",
+ "roc": "Cacgia Roglai",
+ "rod": "Rogo",
+ "roe": "Ronji",
+ "rof": "Rombo",
+ "rog": "Northern Roglai",
+ "rol": "Romblomanon",
+ "rom": "Romany",
+ "roo": "Rotokas",
+ "rop": "Kriol",
+ "ror": "Rongga",
+ "rou": "Runga",
+ "row": "Dela-Oenale",
+ "rpn": "Repanbitip",
+ "rpt": "Rapting",
+ "rri": "Ririo",
+ "rro": "Waima",
+ "rrt": "Arritinngithigh",
+ "rsb": "Romano-Serbian",
+ "rsk": "Ruthenian; Rusyn",
+ "rsl": "Russian Sign Language",
+ "rsm": "Miriwoong Sign Language",
+ "rsn": "Rwandan Sign Language",
+ "rtc": "Rungtu Chin",
+ "rth": "Ratahan",
+ "rtm": "Rotuman",
+ "rts": "Yurats",
+ "rtw": "Rathawi",
+ "ru": "Russian",
+ "rub": "Gungu",
+ "ruc": "Ruuli",
+ "rue": "Rusyn",
+ "ruf": "Luguru",
+ "rug": "Roviana",
+ "ruh": "Ruga",
+ "rui": "Rufiji",
+ "ruk": "Che",
+ "ruo": "Istro Romanian",
+ "rup": "Macedo-Romanian; Aromanian; Arumanian",
+ "ruq": "Megleno Romanian",
+ "rut": "Rutul",
+ "ruu": "Lanas Lobu",
+ "ruy": "Mala (Nigeria)",
+ "ruz": "Ruma",
+ "rw": "Kinyarwanda",
+ "rwa": "Rawo",
+ "rwk": "Rwa",
+ "rwl": "Ruwila",
+ "rwm": "Amba (Uganda)",
+ "rwo": "Rawa",
+ "rwr": "Marwari (India)",
+ "rxd": "Ngardi",
+ "rxw": "Karuwali; Garuwali",
+ "ryn": "Northern Amami-Oshima",
+ "rys": "Yaeyama",
+ "ryu": "Central Okinawan",
+ "rzh": "Rāziḥī",
+ "sa": "Sanskrit",
+ "saa": "Saba",
+ "sab": "Buglere",
+ "sac": "Meskwaki",
+ "sad": "Sandawe",
+ "sae": "Sabanê",
+ "saf": "Safaliba",
+ "sah": "Yakut",
+ "sai": "South American Indian languages",
+ "saj": "Sahu",
+ "sak": "Sake",
+ "sal": "Salishan languages",
+ "sam": "Samaritan Aramaic",
+ "sao": "Sause",
+ "saq": "Samburu",
+ "sar": "Saraveca",
+ "sas": "Sasak",
+ "sat": "Santali",
+ "sau": "Saleman",
+ "sav": "Saafi-Saafi",
+ "saw": "Sawi",
+ "sax": "Sa",
+ "say": "Saya",
+ "saz": "Saurashtra",
+ "sba": "Ngambay",
+ "sbb": "Simbo",
+ "sbc": "Kele (Papua New Guinea)",
+ "sbd": "Southern Samo",
+ "sbe": "Saliba",
+ "sbf": "Chabu; Shabo",
+ "sbg": "Seget",
+ "sbh": "Sori-Harengan",
+ "sbi": "Seti",
+ "sbj": "Surbakhal",
+ "sbk": "Safwa",
+ "sbl": "Botolan Sambal",
+ "sbm": "Sagala",
+ "sbn": "Sindhi Bhil",
+ "sbo": "Sabüm",
+ "sbp": "Sangu (Tanzania)",
+ "sbq": "Sileibi",
+ "sbr": "Sembakung Murut",
+ "sbs": "Subiya",
+ "sbt": "Kimki",
+ "sbu": "Stod Bhoti",
+ "sbv": "Sabine",
+ "sbw": "Simba",
+ "sbx": "Seberuang",
+ "sby": "Soli",
+ "sbz": "Sara Kaba",
+ "sc": "Sardinian",
+ "scb": "Chut",
+ "sce": "Dongxiang",
+ "scf": "San Miguel Creole French",
+ "scg": "Sanggau",
+ "sch": "Sakachep",
+ "sci": "Sri Lankan Creole Malay",
+ "sck": "Sadri",
+ "scl": "Shina",
+ "scn": "Sicilian",
+ "sco": "Scots",
+ "scp": "Hyolmo; Helambu Sherpa",
+ "scq": "Sa'och",
+ "scs": "North Slavey",
+ "sct": "Southern Katang",
+ "scu": "Shumcho",
+ "scv": "Sheni",
+ "scw": "Sha",
+ "scx": "Sicel",
+ "sd": "Sindhi",
+ "sda": "Toraja-Sa'dan",
+ "sdb": "Shabak",
+ "sdc": "Sassarese Sardinian",
+ "sde": "Surubu",
+ "sdf": "Sarli",
+ "sdg": "Savi",
+ "sdh": "Southern Kurdish",
+ "sdj": "Suundi",
+ "sdk": "Sos Kundi",
+ "sdl": "Saudi Arabian Sign Language",
+ "sdn": "Gallurese Sardinian",
+ "sdo": "Bukar-Sadung Bidayuh",
+ "sdp": "Sherdukpen",
+ "sdq": "Semandang",
+ "sdr": "Oraon Sadri",
+ "sds": "Sened",
+ "sdt": "Shuadit",
+ "sdu": "Sarudu",
+ "sdv": "Eastern Sudanic languages",
+ "sdx": "Sibu Melanau",
+ "sdz": "Sallands",
+ "se": "Northern Sami",
+ "sea": "Semai",
+ "seb": "Shempire Senoufo",
+ "sec": "Sechelt",
+ "sed": "Sedang",
+ "see": "Seneca",
+ "sef": "Cebaara Senoufo",
+ "seg": "Segeju",
+ "seh": "Sena",
+ "sei": "Seri",
+ "sej": "Sene",
+ "sek": "Sekani",
+ "sel": "Selkup",
+ "sem": "Semitic languages",
+ "sen": "Nanerigé Sénoufo",
+ "seo": "Suarmin",
+ "sep": "Sìcìté Sénoufo",
+ "seq": "Senara Sénoufo",
+ "ser": "Serrano",
+ "ses": "Koyraboro Senni Songhai",
+ "set": "Sentani",
+ "seu": "Serui-Laut",
+ "sev": "Nyarafolo Senoufo",
+ "sew": "Sewa Bay",
+ "sey": "Secoya",
+ "sez": "Senthang Chin",
+ "sfb": "Langue des signes de Belgique Francophone; French Belgian Sign Language",
+ "sfe": "Eastern Subanen",
+ "sfm": "Small Flowery Miao",
+ "sfs": "South African Sign Language",
+ "sfw": "Sehwi",
+ "sg": "Sango",
+ "sga": "Old Irish (to 900)",
+ "sgb": "Mag-antsi Ayta",
+ "sgc": "Kipsigis",
+ "sgd": "Surigaonon",
+ "sge": "Segai",
+ "sgg": "Swiss-German Sign Language",
+ "sgh": "Shughni",
+ "sgi": "Suga",
+ "sgj": "Surgujia",
+ "sgk": "Sangkong",
+ "sgm": "Singa",
+ "sgn": "Sign languages",
+ "sgp": "Singpho",
+ "sgr": "Sangisari",
+ "sgs": "Samogitian",
+ "sgt": "Brokpake",
+ "sgu": "Salas",
+ "sgw": "Sebat Bet Gurage",
+ "sgx": "Sierra Leone Sign Language",
+ "sgy": "Sanglechi",
+ "sgz": "Sursurunga",
+ "sh": "Serbo-Croatian",
+ "sha": "Shall-Zwall",
+ "shb": "Ninam",
+ "shc": "Sonde",
+ "shd": "Kundal Shahi",
+ "she": "Sheko",
+ "shg": "Shua",
+ "shh": "Shoshoni",
+ "shi": "Tachelhit",
+ "shj": "Shatt",
+ "shk": "Shilluk",
+ "shl": "Shendu",
+ "shm": "Shahrudi",
+ "shn": "Shan",
+ "sho": "Shanga",
+ "shp": "Shipibo-Conibo",
+ "shq": "Sala",
+ "shr": "Shi",
+ "shs": "Shuswap",
+ "sht": "Shasta",
+ "shu": "Chadian Arabic",
+ "shv": "Shehri",
+ "shw": "Shwai",
+ "shx": "She",
+ "shy": "Tachawit",
+ "shz": "Syenara Senoufo",
+ "si": "Sinhala; Sinhalese",
+ "sia": "Akkala Sami",
+ "sib": "Sebop",
+ "sid": "Sidamo",
+ "sie": "Simaa",
+ "sif": "Siamou",
+ "sig": "Paasaal",
+ "sih": "Zire; Sîshëë",
+ "sii": "Shom Peng",
+ "sij": "Numbami",
+ "sik": "Sikiana",
+ "sil": "Tumulung Sisaala",
+ "sim": "Mende (Papua New Guinea)",
+ "sio": "Siouan languages",
+ "sip": "Sikkimese",
+ "siq": "Sonia",
+ "sir": "Siri",
+ "sis": "Siuslaw",
+ "sit": "Sino-Tibetan languages",
+ "siu": "Sinagen",
+ "siv": "Sumariup",
+ "siw": "Siwai",
+ "six": "Sumau",
+ "siy": "Sivandi",
+ "siz": "Siwi",
+ "sja": "Epena",
+ "sjb": "Sajau Basap",
+ "sjd": "Kildin Sami",
+ "sje": "Pite Sami",
+ "sjg": "Assangori",
+ "sjk": "Kemi Sami",
+ "sjl": "Sajalong; Miji",
+ "sjm": "Mapun",
+ "sjn": "Sindarin",
+ "sjo": "Xibe",
+ "sjp": "Surjapuri",
+ "sjr": "Siar-Lak",
+ "sjs": "Senhaja De Srair",
+ "sjt": "Ter Sami",
+ "sju": "Ume Sami",
+ "sjw": "Shawnee",
+ "sk": "Slovak",
+ "ska": "Skagit",
+ "skb": "Saek",
+ "skc": "Ma Manda",
+ "skd": "Southern Sierra Miwok",
+ "ske": "Seke (Vanuatu)",
+ "skf": "Sakirabiá",
+ "skg": "Sakalava Malagasy",
+ "skh": "Sikule",
+ "ski": "Sika",
+ "skj": "Seke (Nepal)",
+ "skm": "Kutong",
+ "skn": "Kolibugan Subanon",
+ "sko": "Seko Tengah",
+ "skp": "Sekapan",
+ "skq": "Sininkere",
+ "skr": "Saraiki; Seraiki",
+ "sks": "Maia",
+ "skt": "Sakata",
+ "sku": "Sakao",
+ "skv": "Skou",
+ "skw": "Skepi Creole Dutch",
+ "skx": "Seko Padang",
+ "sky": "Sikaiana",
+ "skz": "Sekar",
+ "sl": "Slovenian",
+ "sla": "Slavic languages",
+ "slc": "Sáliba",
+ "sld": "Sissala",
+ "sle": "Sholaga",
+ "slf": "Swiss-Italian Sign Language",
+ "slg": "Selungai Murut",
+ "slh": "Southern Puget Sound Salish",
+ "sli": "Lower Silesian",
+ "slj": "Salumá",
+ "sll": "Salt-Yui",
+ "slm": "Pangutaran Sama",
+ "sln": "Salinan",
+ "slp": "Lamaholot",
+ "slq": "Salchuq",
+ "slr": "Salar",
+ "sls": "Singapore Sign Language",
+ "slt": "Sila",
+ "slu": "Selaru",
+ "slw": "Sialum",
+ "slx": "Salampasu",
+ "sly": "Selayar",
+ "slz": "Ma'ya",
+ "sm": "Samoan",
+ "sma": "Southern Sami",
+ "smb": "Simbari",
+ "smc": "Som",
+ "smf": "Auwe",
+ "smg": "Simbali",
+ "smh": "Samei",
+ "smi": "Sami languages",
+ "smj": "Lule Sami",
+ "smk": "Bolinao",
+ "sml": "Central Sama",
+ "smm": "Musasa",
+ "smn": "Inari Sami",
+ "smp": "Samaritan",
+ "smq": "Samo",
+ "smr": "Simeulue",
+ "sms": "Skolt Sami",
+ "smt": "Simte",
+ "smu": "Somray",
+ "smv": "Samvedi",
+ "smw": "Sumbawa",
+ "smx": "Samba",
+ "smy": "Semnani",
+ "smz": "Simeku",
+ "sn": "Shona",
+ "snc": "Sinaugoro",
+ "sne": "Bau Bidayuh",
+ "snf": "Noon",
+ "sng": "Sanga (Democratic Republic of Congo)",
+ "sni": "Sensi",
+ "snj": "Riverain Sango",
+ "snk": "Soninke",
+ "snl": "Sangil",
+ "snm": "Southern Ma'di",
+ "snn": "Siona",
+ "sno": "Snohomish",
+ "snp": "Siane",
+ "snq": "Sangu (Gabon)",
+ "snr": "Sihan",
+ "sns": "South West Bay; Nahavaq",
+ "snu": "Senggi; Viid",
+ "snv": "Sa'ban",
+ "snw": "Selee",
+ "snx": "Sam",
+ "sny": "Saniyo-Hiyewe",
+ "snz": "Kou",
+ "so": "Somali",
+ "soa": "Thai Song",
+ "sob": "Sobei",
+ "soc": "So (Democratic Republic of Congo)",
+ "sod": "Songoora",
+ "soe": "Songomeno",
+ "sog": "Sogdian",
+ "soh": "Aka",
+ "soi": "Sonha",
+ "soj": "Soi",
+ "sok": "Sokoro",
+ "sol": "Solos",
+ "son": "Songhai languages",
+ "soo": "Songo",
+ "sop": "Songe",
+ "soq": "Kanasi",
+ "sor": "Somrai",
+ "sos": "Seeku",
+ "sou": "Southern Thai",
+ "sov": "Sonsorol",
+ "sow": "Sowanda",
+ "sox": "Swo",
+ "soy": "Miyobe",
+ "soz": "Temi",
+ "spb": "Sepa (Indonesia)",
+ "spc": "Sapé",
+ "spd": "Saep",
+ "spe": "Sepa (Papua New Guinea)",
+ "spg": "Sian",
+ "spi": "Saponi",
+ "spk": "Sengo",
+ "spl": "Selepet",
+ "spm": "Akukem",
+ "spn": "Sanapaná",
+ "spo": "Spokane",
+ "spp": "Supyire Senoufo",
+ "spq": "Loreto-Ucayali Spanish",
+ "spr": "Saparua",
+ "sps": "Saposa",
+ "spt": "Spiti Bhoti",
+ "spu": "Sapuan",
+ "spv": "Sambalpuri; Kosli",
+ "spx": "South Picene",
+ "spy": "Sabaot",
+ "sq": "Albanian",
+ "sqa": "Shama-Sambuga",
+ "sqh": "Shau",
+ "sqj": "Albanian languages",
+ "sqk": "Albanian Sign Language",
+ "sqm": "Suma",
+ "sqn": "Susquehannock",
+ "sqo": "Sorkhei",
+ "sqq": "Sou",
+ "sqr": "Siculo Arabic",
+ "sqs": "Sri Lankan Sign Language",
+ "sqt": "Soqotri",
+ "squ": "Squamish",
+ "sqx": "Kufr Qassem Sign Language (KQSL)",
+ "sr": "Serbian",
+ "sra": "Saruga",
+ "srb": "Sora",
+ "src": "Logudorese Sardinian",
+ "sre": "Sara",
+ "srf": "Nafi",
+ "srg": "Sulod",
+ "srh": "Sarikoli",
+ "sri": "Siriano",
+ "srk": "Serudung Murut",
+ "srl": "Isirawa",
+ "srm": "Saramaccan",
+ "srn": "Sranan Tongo",
+ "sro": "Campidanese Sardinian",
+ "srq": "Sirionó",
+ "srr": "Serer",
+ "srs": "Sarsi",
+ "srt": "Sauri",
+ "sru": "Suruí",
+ "srv": "Southern Sorsoganon",
+ "srw": "Serua",
+ "srx": "Sirmauri",
+ "sry": "Sera",
+ "srz": "Shahmirzadi",
+ "ss": "Swati",
+ "ssa": "Nilo-Saharan languages",
+ "ssb": "Southern Sama",
+ "ssc": "Suba-Simbiti",
+ "ssd": "Siroi",
+ "sse": "Balangingi; Bangingih Sama",
+ "ssf": "Thao",
+ "ssg": "Seimat",
+ "ssh": "Shihhi Arabic",
+ "ssi": "Sansi",
+ "ssj": "Sausi",
+ "ssk": "Sunam",
+ "ssl": "Western Sisaala",
+ "ssm": "Semnam",
+ "ssn": "Waata",
+ "sso": "Sissano",
+ "ssp": "Spanish Sign Language",
+ "ssq": "So'a",
+ "ssr": "Swiss-French Sign Language",
+ "sss": "Sô",
+ "sst": "Sinasina",
+ "ssu": "Susuami",
+ "ssv": "Shark Bay",
+ "ssx": "Samberigi",
+ "ssy": "Saho",
+ "ssz": "Sengseng",
+ "st": "Southern Sotho",
+ "sta": "Settla",
+ "stb": "Northern Subanen",
+ "std": "Sentinel",
+ "ste": "Liana-Seti",
+ "stf": "Seta",
+ "stg": "Trieng",
+ "sth": "Shelta",
+ "sti": "Bulo Stieng",
+ "stj": "Matya Samo",
+ "stk": "Arammba",
+ "stl": "Stellingwerfs",
+ "stm": "Setaman",
+ "stn": "Owa",
+ "sto": "Stoney",
+ "stp": "Southeastern Tepehuan",
+ "stq": "Saterfriesisch",
+ "str": "Straits Salish",
+ "sts": "Shumashti",
+ "stt": "Budeh Stieng",
+ "stu": "Samtao",
+ "stv": "Silt'e",
+ "stw": "Satawalese",
+ "sty": "Siberian Tatar",
+ "su": "Sundanese",
+ "sua": "Sulka",
+ "sub": "Suku",
+ "suc": "Western Subanon",
+ "sue": "Suena",
+ "sug": "Suganga",
+ "sui": "Suki",
+ "suj": "Shubi",
+ "suk": "Sukuma",
+ "suo": "Bouni",
+ "suq": "Tirmaga-Chai Suri; Suri",
+ "sur": "Mwaghavul",
+ "sus": "Susu",
+ "sut": "Subtiaba",
+ "suv": "Puroik",
+ "suw": "Sumbwa",
+ "sux": "Sumerian",
+ "suy": "Suyá",
+ "suz": "Sunwar",
+ "sv": "Swedish",
+ "sva": "Svan",
+ "svb": "Ulau-Suain",
+ "svc": "Vincentian Creole English",
+ "sve": "Serili",
+ "svk": "Slovakian Sign Language",
+ "svm": "Slavomolisano",
+ "svs": "Savosavo",
+ "svx": "Skalvian",
+ "sw": "Swahili (macrolanguage)",
+ "swb": "Maore Comorian",
+ "swc": "Congo Swahili",
+ "swf": "Sere",
+ "swg": "Swabian",
+ "swh": "Swahili (individual language); Kiswahili",
+ "swi": "Sui",
+ "swj": "Sira",
+ "swk": "Malawi Sena",
+ "swl": "Swedish Sign Language",
+ "swm": "Samosa",
+ "swn": "Sawknah",
+ "swo": "Shanenawa",
+ "swp": "Suau",
+ "swq": "Sharwa",
+ "swr": "Saweru",
+ "sws": "Seluwasan",
+ "swt": "Sawila",
+ "swu": "Suwawa",
+ "swv": "Shekhawati",
+ "sww": "Sowa",
+ "swx": "Suruahá",
+ "swy": "Sarua",
+ "sxb": "Suba",
+ "sxc": "Sicanian",
+ "sxe": "Sighu",
+ "sxg": "Shuhi; Shixing",
+ "sxk": "Southern Kalapuya",
+ "sxl": "Selian",
+ "sxm": "Samre",
+ "sxn": "Sangir",
+ "sxo": "Sorothaptic",
+ "sxr": "Saaroa",
+ "sxs": "Sasaru",
+ "sxu": "Upper Saxon",
+ "sxw": "Saxwe Gbe",
+ "sya": "Siang",
+ "syb": "Central Subanen",
+ "syc": "Classical Syriac",
+ "syd": "Samoyedic languages",
+ "syi": "Seki",
+ "syk": "Sukur",
+ "syl": "Sylheti",
+ "sym": "Maya Samo",
+ "syn": "Senaya",
+ "syo": "Suoy",
+ "syr": "Syriac",
+ "sys": "Sinyar",
+ "syw": "Kagate",
+ "syx": "Samay",
+ "syy": "Al-Sayyid Bedouin Sign Language",
+ "sza": "Semelai",
+ "szb": "Ngalum",
+ "szc": "Semaq Beri",
+ "szd": "Seru",
+ "sze": "Seze",
+ "szg": "Sengele",
+ "szl": "Silesian",
+ "szn": "Sula",
+ "szp": "Suabo",
+ "szs": "Solomon Islands Sign Language",
+ "szv": "Isu (Fako Division)",
+ "szw": "Sawai",
+ "szy": "Sakizaya",
+ "ta": "Tamil",
+ "taa": "Lower Tanana",
+ "tab": "Tabassaran",
+ "tac": "Lowland Tarahumara",
+ "tad": "Tause",
+ "tae": "Tariana",
+ "taf": "Tapirapé",
+ "tag": "Tagoi",
+ "tai": "Tai languages",
+ "taj": "Eastern Tamang",
+ "tak": "Tala",
+ "tal": "Tal",
+ "tan": "Tangale",
+ "tao": "Yami",
+ "tap": "Taabwa",
+ "taq": "Tamasheq",
+ "tar": "Central Tarahumara",
+ "tas": "Tay Boi",
+ "tau": "Upper Tanana",
+ "tav": "Tatuyo",
+ "taw": "Tai",
+ "tax": "Tamki",
+ "tay": "Atayal",
+ "taz": "Tocho",
+ "tba": "Aikanã",
+ "tbc": "Takia",
+ "tbd": "Kaki Ae",
+ "tbe": "Tanimbili",
+ "tbf": "Mandara",
+ "tbg": "North Tairora",
+ "tbh": "Dharawal; Thurawal",
+ "tbi": "Gaam",
+ "tbj": "Tiang",
+ "tbk": "Calamian Tagbanwa",
+ "tbl": "Tboli",
+ "tbm": "Tagbu",
+ "tbn": "Barro Negro Tunebo",
+ "tbo": "Tawala",
+ "tbp": "Taworta; Diebroud",
+ "tbq": "Tibeto-Burman languages",
+ "tbr": "Tumtum",
+ "tbs": "Tanguat",
+ "tbt": "Tembo (Kitembo)",
+ "tbu": "Tubar",
+ "tbv": "Tobo",
+ "tbw": "Tagbanwa",
+ "tbx": "Kapin",
+ "tby": "Tabaru",
+ "tbz": "Ditammari",
+ "tca": "Ticuna",
+ "tcb": "Tanacross",
+ "tcc": "Datooga",
+ "tcd": "Tafi",
+ "tce": "Southern Tutchone",
+ "tcf": "Malinaltepec Me'phaa; Malinaltepec Tlapanec",
+ "tcg": "Tamagario",
+ "tch": "Turks And Caicos Creole English",
+ "tci": "Wára",
+ "tck": "Tchitchege",
+ "tcl": "Taman (Myanmar)",
+ "tcm": "Tanahmerah",
+ "tcn": "Tichurong",
+ "tco": "Taungyo",
+ "tcp": "Tawr Chin",
+ "tcq": "Kaiy",
+ "tcs": "Torres Strait Creole; Yumplatok",
+ "tct": "T'en",
+ "tcu": "Southeastern Tarahumara",
+ "tcw": "Tecpatlán Totonac",
+ "tcx": "Toda",
+ "tcy": "Tulu",
+ "tcz": "Thado Chin",
+ "tda": "Tagdal",
+ "tdb": "Panchpargania",
+ "tdc": "Emberá-Tadó",
+ "tdd": "Tai Nüa",
+ "tde": "Tiranige Diga Dogon",
+ "tdf": "Talieng",
+ "tdg": "Western Tamang",
+ "tdh": "Thulung",
+ "tdi": "Tomadino",
+ "tdj": "Tajio",
+ "tdk": "Tambas",
+ "tdl": "Sur",
+ "tdm": "Taruma",
+ "tdn": "Tondano",
+ "tdo": "Teme",
+ "tdq": "Tita",
+ "tdr": "Todrah",
+ "tds": "Doutai",
+ "tdt": "Tetun Dili",
+ "tdv": "Toro",
+ "tdx": "Tandroy-Mahafaly Malagasy",
+ "tdy": "Tadyawan",
+ "te": "Telugu",
+ "tea": "Temiar",
+ "teb": "Tetete",
+ "tec": "Terik",
+ "ted": "Tepo Krumen",
+ "tee": "Huehuetla Tepehua",
+ "tef": "Teressa",
+ "teg": "Teke-Tege",
+ "teh": "Tehuelche",
+ "tei": "Torricelli",
+ "tek": "Ibali Teke",
+ "tem": "Timne",
+ "ten": "Tama (Colombia)",
+ "teo": "Teso",
+ "tep": "Tepecano",
+ "teq": "Temein",
+ "ter": "Tereno",
+ "tes": "Tengger",
+ "tet": "Tetum",
+ "teu": "Soo",
+ "tev": "Teor",
+ "tew": "Tewa (USA)",
+ "tex": "Tennet",
+ "tey": "Tulishi",
+ "tez": "Tetserret",
+ "tfi": "Tofin Gbe",
+ "tfn": "Tanaina",
+ "tfo": "Tefaro",
+ "tfr": "Teribe",
+ "tft": "Ternate",
+ "tg": "Tajik",
+ "tga": "Sagalla",
+ "tgb": "Tobilung",
+ "tgc": "Tigak",
+ "tgd": "Ciwogai",
+ "tge": "Eastern Gorkha Tamang",
+ "tgf": "Chalikha",
+ "tgh": "Tobagonian Creole English",
+ "tgi": "Lawunuia",
+ "tgj": "Tagin",
+ "tgn": "Tandaganon",
+ "tgo": "Sudest",
+ "tgp": "Tangoa",
+ "tgq": "Tring",
+ "tgr": "Tareng",
+ "tgs": "Nume",
+ "tgt": "Central Tagbanwa",
+ "tgu": "Tanggu",
+ "tgv": "Tingui-Boto",
+ "tgw": "Tagwana Senoufo",
+ "tgx": "Tagish",
+ "tgy": "Togoyo",
+ "tgz": "Tagalaka",
+ "th": "Thai",
+ "thd": "Kuuk Thaayorre; Thayore",
+ "the": "Chitwania Tharu",
+ "thf": "Thangmi",
+ "thh": "Northern Tarahumara",
+ "thi": "Tai Long",
+ "thk": "Tharaka; Kitharaka",
+ "thl": "Dangaura Tharu",
+ "thm": "Aheu",
+ "thn": "Thachanadan",
+ "thp": "Thompson",
+ "thq": "Kochila Tharu",
+ "thr": "Rana Tharu",
+ "ths": "Thakali",
+ "tht": "Tahltan",
+ "thu": "Thuri",
+ "thv": "Tahaggart Tamahaq",
+ "thy": "Tha",
+ "thz": "Tayart Tamajeq",
+ "ti": "Tigrinya",
+ "tia": "Tidikelt Tamazight",
+ "tic": "Tira",
+ "tif": "Tifal",
+ "tig": "Tigre",
+ "tih": "Timugon Murut",
+ "tii": "Tiene",
+ "tij": "Tilung",
+ "tik": "Tikar",
+ "til": "Tillamook",
+ "tim": "Timbe",
+ "tin": "Tindi",
+ "tio": "Teop",
+ "tip": "Trimuris",
+ "tiq": "Tiéfo",
+ "tis": "Masadiit Itneg",
+ "tit": "Tinigua",
+ "tiu": "Adasen",
+ "tiv": "Tiv",
+ "tiw": "Tiwi",
+ "tix": "Southern Tiwa",
+ "tiy": "Tiruray",
+ "tiz": "Tai Hongjin",
+ "tja": "Tajuasohn",
+ "tjg": "Tunjung",
+ "tji": "Northern Tujia",
+ "tjj": "Tjungundji",
+ "tjl": "Tai Laing",
+ "tjm": "Timucua",
+ "tjn": "Tonjon",
+ "tjo": "Temacine Tamazight",
+ "tjp": "Tjupany",
+ "tjs": "Southern Tujia",
+ "tju": "Tjurruru",
+ "tjw": "Djabwurrung",
+ "tk": "Turkmen",
+ "tka": "Truká",
+ "tkb": "Buksa",
+ "tkd": "Tukudede",
+ "tke": "Takwane",
+ "tkf": "Tukumanféd",
+ "tkg": "Tesaka Malagasy",
+ "tkl": "Tokelau",
+ "tkm": "Takelma",
+ "tkn": "Toku-No-Shima",
+ "tkp": "Tikopia",
+ "tkq": "Tee",
+ "tkr": "Tsakhur",
+ "tks": "Takestani",
+ "tkt": "Kathoriya Tharu",
+ "tku": "Upper Necaxa Totonac",
+ "tkv": "Mur Pano",
+ "tkw": "Teanu",
+ "tkx": "Tangko",
+ "tkz": "Takua",
+ "tl": "Tagalog",
+ "tla": "Southwestern Tepehuan",
+ "tlb": "Tobelo",
+ "tlc": "Yecuatla Totonac",
+ "tld": "Talaud",
+ "tlf": "Telefol",
+ "tlg": "Tofanma",
+ "tlh": "Klingon; tlhIngan Hol",
+ "tli": "Tlingit",
+ "tlj": "Talinga-Bwisi",
+ "tlk": "Taloki",
+ "tll": "Tetela",
+ "tlm": "Tolomako",
+ "tln": "Talondo'",
+ "tlo": "Talodi",
+ "tlp": "Filomena Mata-Coahuitlán Totonac",
+ "tlq": "Tai Loi",
+ "tlr": "Talise",
+ "tls": "Tambotalo",
+ "tlt": "Sou Nama; Teluti",
+ "tlu": "Tulehu",
+ "tlv": "Taliabu",
+ "tlx": "Khehek",
+ "tly": "Talysh",
+ "tma": "Tama (Chad)",
+ "tmb": "Katbol; Avava",
+ "tmc": "Tumak",
+ "tmd": "Haruai",
+ "tme": "Tremembé",
+ "tmf": "Toba-Maskoy",
+ "tmg": "Ternateño",
+ "tmh": "Tamashek",
+ "tmi": "Tutuba",
+ "tmj": "Samarokena",
+ "tmk": "Northwestern Tamang",
+ "tml": "Tamnim Citak",
+ "tmm": "Tai Thanh",
+ "tmn": "Taman (Indonesia)",
+ "tmo": "Temoq",
+ "tmq": "Tumleo",
+ "tmr": "Jewish Babylonian Aramaic (ca. 200-1200 CE)",
+ "tms": "Tima",
+ "tmt": "Tasmate",
+ "tmu": "Iau",
+ "tmv": "Tembo (Motembo)",
+ "tmw": "Temuan",
+ "tmy": "Tami",
+ "tmz": "Tamanaku",
+ "tn": "Tswana",
+ "tna": "Tacana",
+ "tnb": "Western Tunebo",
+ "tnc": "Tanimuca-Retuarã",
+ "tnd": "Angosturas Tunebo",
+ "tng": "Tobanga",
+ "tnh": "Maiani",
+ "tni": "Tandia",
+ "tnk": "Kwamera",
+ "tnl": "Lenakel",
+ "tnm": "Tabla",
+ "tnn": "North Tanna",
+ "tno": "Toromono",
+ "tnp": "Whitesands",
+ "tnq": "Taino",
+ "tnr": "Ménik",
+ "tns": "Tenis",
+ "tnt": "Tontemboan",
+ "tnu": "Tay Khang",
+ "tnv": "Tangchangya",
+ "tnw": "Tonsawang",
+ "tnx": "Tanema",
+ "tny": "Tongwe",
+ "tnz": "Ten'edn",
+ "to": "Tonga (Tonga Islands)",
+ "tob": "Toba",
+ "toc": "Coyutla Totonac",
+ "tod": "Toma",
+ "tof": "Gizrra",
+ "tog": "Tonga (Nyasa)",
+ "toh": "Gitonga",
+ "toi": "Tonga (Zambia)",
+ "toj": "Tojolabal",
+ "tok": "Toki Pona",
+ "tol": "Tolowa",
+ "tom": "Tombulu",
+ "too": "Xicotepec De Juárez Totonac",
+ "top": "Papantla Totonac",
+ "toq": "Toposa",
+ "tor": "Togbo-Vara Banda",
+ "tos": "Highland Totonac",
+ "tou": "Tho",
+ "tov": "Upper Taromi",
+ "tow": "Jemez",
+ "tox": "Tobian",
+ "toy": "Topoiyo",
+ "toz": "To",
+ "tpa": "Taupota",
+ "tpc": "Azoyú Me'phaa; Azoyú Tlapanec",
+ "tpe": "Tippera",
+ "tpf": "Tarpia",
+ "tpg": "Kula",
+ "tpi": "Tok Pisin",
+ "tpj": "Tapieté",
+ "tpk": "Tupinikin",
+ "tpl": "Tlacoapa Me'phaa; Tlacoapa Tlapanec",
+ "tpm": "Tampulma",
+ "tpn": "Tupinambá",
+ "tpo": "Tai Pao",
+ "tpp": "Pisaflores Tepehua",
+ "tpq": "Tukpa",
+ "tpr": "Tuparí",
+ "tpt": "Tlachichilco Tepehua",
+ "tpu": "Tampuan",
+ "tpv": "Tanapag",
+ "tpw": "Tupí",
+ "tpx": "Acatepec Me'phaa; Acatepec Tlapanec",
+ "tpy": "Trumai",
+ "tpz": "Tinputz",
+ "tqb": "Tembé",
+ "tql": "Lehali",
+ "tqm": "Turumsa",
+ "tqn": "Tenino",
+ "tqo": "Toaripi",
+ "tqp": "Tomoip",
+ "tqq": "Tunni",
+ "tqr": "Torona",
+ "tqt": "Western Totonac",
+ "tqu": "Touo",
+ "tqw": "Tonkawa",
+ "tr": "Turkish",
+ "tra": "Tirahi",
+ "trb": "Terebu",
+ "trc": "Copala Triqui",
+ "trd": "Turi",
+ "tre": "East Tarangan",
+ "trf": "Trinidadian Creole English",
+ "trg": "Lishán Didán",
+ "trh": "Turaka",
+ "tri": "Trió",
+ "trj": "Toram",
+ "trk": "Turkic languages",
+ "trl": "Traveller Scottish",
+ "trm": "Tregami",
+ "trn": "Trinitario",
+ "tro": "Tarao Naga",
+ "trp": "Kok Borok",
+ "trq": "San Martín Itunyoso Triqui",
+ "trr": "Taushiro",
+ "trs": "Chicahuaxtla Triqui",
+ "trt": "Tunggare",
+ "tru": "Turoyo; Surayt",
+ "trv": "Sediq; Seediq; Taroko",
+ "trw": "Torwali",
+ "trx": "Tringgus-Sembaan Bidayuh",
+ "try": "Turung",
+ "trz": "Torá",
+ "ts": "Tsonga",
+ "tsa": "Tsaangi",
+ "tsb": "Tsamai",
+ "tsc": "Tswa",
+ "tsd": "Tsakonian",
+ "tse": "Tunisian Sign Language",
+ "tsg": "Tausug",
+ "tsh": "Tsuvan",
+ "tsi": "Tsimshian",
+ "tsj": "Tshangla",
+ "tsk": "Tseku",
+ "tsl": "Ts'ün-Lao",
+ "tsm": "Turkish Sign Language; Türk İşaret Dili",
+ "tsp": "Northern Toussian",
+ "tsq": "Thai Sign Language",
+ "tsr": "Akei",
+ "tss": "Taiwan Sign Language",
+ "tst": "Tondi Songway Kiini",
+ "tsu": "Tsou",
+ "tsv": "Tsogo",
+ "tsw": "Tsishingini",
+ "tsx": "Mubami",
+ "tsy": "Tebul Sign Language",
+ "tsz": "Purepecha",
+ "tt": "Tatar",
+ "tta": "Tutelo",
+ "ttb": "Gaa",
+ "ttc": "Tektiteko",
+ "ttd": "Tauade",
+ "tte": "Bwanabwana",
+ "ttf": "Tuotomb",
+ "ttg": "Tutong",
+ "tth": "Upper Ta'oih",
+ "tti": "Tobati",
+ "ttj": "Tooro",
+ "ttk": "Totoro",
+ "ttl": "Totela",
+ "ttm": "Northern Tutchone",
+ "ttn": "Towei",
+ "tto": "Lower Ta'oih",
+ "ttp": "Tombelala",
+ "ttq": "Tawallammat Tamajaq",
+ "ttr": "Tera",
+ "tts": "Northeastern Thai",
+ "ttt": "Muslim Tat",
+ "ttu": "Torau",
+ "ttv": "Titan",
+ "ttw": "Long Wat",
+ "tty": "Sikaritai",
+ "ttz": "Tsum",
+ "tua": "Wiarumus",
+ "tub": "Tübatulabal",
+ "tuc": "Mutu",
+ "tud": "Tuxá",
+ "tue": "Tuyuca",
+ "tuf": "Central Tunebo",
+ "tug": "Tunia",
+ "tuh": "Taulil",
+ "tui": "Tupuri",
+ "tuj": "Tugutil",
+ "tul": "Tula",
+ "tum": "Tumbuka",
+ "tun": "Tunica",
+ "tuo": "Tucano",
+ "tup": "Tupi languages",
+ "tuq": "Tedaga",
+ "tus": "Tuscarora",
+ "tut": "Altaic languages",
+ "tuu": "Tututni",
+ "tuv": "Turkana",
+ "tuw": "Tungus languages",
+ "tux": "Tuxináwa",
+ "tuy": "Tugen",
+ "tuz": "Turka",
+ "tva": "Vaghua",
+ "tvd": "Tsuvadi",
+ "tve": "Te'un",
+ "tvk": "Southeast Ambrym",
+ "tvl": "Tuvalu",
+ "tvm": "Tela-Masbuar",
+ "tvn": "Tavoyan",
+ "tvo": "Tidore",
+ "tvs": "Taveta",
+ "tvt": "Tutsa Naga",
+ "tvu": "Tunen",
+ "tvw": "Sedoa",
+ "tvx": "Taivoan",
+ "tvy": "Timor Pidgin",
+ "tw": "Twi",
+ "twa": "Twana",
+ "twb": "Western Tawbuid",
+ "twc": "Teshenawa",
+ "twd": "Twents",
+ "twe": "Tewa (Indonesia)",
+ "twf": "Northern Tiwa",
+ "twg": "Tereweng",
+ "twh": "Tai Dón",
+ "twl": "Tawara",
+ "twm": "Tawang Monpa",
+ "twn": "Twendi",
+ "two": "Tswapong",
+ "twp": "Ere",
+ "twq": "Tasawaq",
+ "twr": "Southwestern Tarahumara",
+ "twt": "Turiwára",
+ "twu": "Termanu",
+ "tww": "Tuwari",
+ "twx": "Tewe",
+ "twy": "Tawoyan",
+ "txa": "Tombonuo",
+ "txb": "Tokharian B",
+ "txc": "Tsetsaut",
+ "txe": "Totoli",
+ "txg": "Tangut",
+ "txh": "Thracian",
+ "txi": "Ikpeng",
+ "txj": "Tarjumo",
+ "txm": "Tomini",
+ "txn": "West Tarangan",
+ "txo": "Toto",
+ "txq": "Tii",
+ "txr": "Tartessian",
+ "txs": "Tonsea",
+ "txt": "Citak",
+ "txu": "Kayapó",
+ "txx": "Tatana",
+ "txy": "Tanosy Malagasy",
+ "ty": "Tahitian",
+ "tya": "Tauya",
+ "tye": "Kyanga",
+ "tyh": "O'du",
+ "tyi": "Teke-Tsaayi",
+ "tyj": "Tai Do; Tai Yo",
+ "tyl": "Thu Lao",
+ "tyn": "Kombai",
+ "typ": "Thaypan",
+ "tyr": "Tai Daeng",
+ "tys": "Tày Sa Pa",
+ "tyt": "Tày Tac",
+ "tyu": "Kua",
+ "tyv": "Tuvinian",
+ "tyx": "Teke-Tyee",
+ "tyy": "Tiyaa",
+ "tyz": "Tày",
+ "tza": "Tanzanian Sign Language",
+ "tzh": "Tzeltal",
+ "tzj": "Tz'utujil",
+ "tzl": "Talossan",
+ "tzm": "Central Atlas Tamazight",
+ "tzn": "Tugun",
+ "tzo": "Tzotzil",
+ "tzx": "Tabriak",
+ "uam": "Uamué",
+ "uan": "Kuan",
+ "uar": "Tairuma",
+ "uba": "Ubang",
+ "ubi": "Ubi",
+ "ubl": "Buhi'non Bikol",
+ "ubr": "Ubir",
+ "ubu": "Umbu-Ungu",
+ "uby": "Ubykh",
+ "uda": "Uda",
+ "ude": "Udihe",
+ "udg": "Muduga",
+ "udi": "Udi",
+ "udj": "Ujir",
+ "udl": "Wuzlam",
+ "udm": "Udmurt",
+ "udu": "Uduk",
+ "ues": "Kioko",
+ "ufi": "Ufim",
+ "ug": "Uighur; Uyghur",
+ "uga": "Ugaritic",
+ "ugb": "Kuku-Ugbanh",
+ "uge": "Ughele",
+ "ugh": "Kubachi",
+ "ugn": "Ugandan Sign Language",
+ "ugo": "Ugong",
+ "ugy": "Uruguayan Sign Language",
+ "uha": "Uhami",
+ "uhn": "Damal",
+ "uis": "Uisai",
+ "uiv": "Iyive",
+ "uji": "Tanjijili",
+ "uk": "Ukrainian",
+ "uka": "Kaburi",
+ "ukg": "Ukuriguma",
+ "ukh": "Ukhwejo",
+ "uki": "Kui (India)",
+ "ukk": "Muak Sa-aak",
+ "ukl": "Ukrainian Sign Language",
+ "ukp": "Ukpe-Bayobiri",
+ "ukq": "Ukwa",
+ "uks": "Urubú-Kaapor Sign Language; Kaapor Sign Language",
+ "uku": "Ukue",
+ "ukv": "Kuku",
+ "ukw": "Ukwuani-Aboh-Ndoni",
+ "uky": "Kuuk-Yak",
+ "ula": "Fungwa",
+ "ulb": "Ulukwumi",
+ "ulc": "Ulch",
+ "ule": "Lule",
+ "ulf": "Usku; Afra",
+ "uli": "Ulithian",
+ "ulk": "Meriam Mir",
+ "ull": "Ullatan",
+ "ulm": "Ulumanda'",
+ "uln": "Unserdeutsch",
+ "ulu": "Uma' Lung",
+ "ulw": "Ulwa",
+ "uma": "Umatilla",
+ "umb": "Umbundu",
+ "umc": "Marrucinian",
+ "umd": "Umbindhamu",
+ "umg": "Morrobalama; Umbuygamu",
+ "umi": "Ukit",
+ "umm": "Umon",
+ "umn": "Makyan Naga",
+ "umo": "Umotína",
+ "ump": "Umpila",
+ "umr": "Umbugarla",
+ "ums": "Pendau",
+ "umu": "Munsee",
+ "una": "North Watut",
+ "und": "Undetermined",
+ "une": "Uneme",
+ "ung": "Ngarinyin",
+ "uni": "Uni",
+ "unk": "Enawené-Nawé",
+ "unm": "Unami",
+ "unn": "Kurnai",
+ "unr": "Mundari",
+ "unu": "Unubahe",
+ "unx": "Munda",
+ "unz": "Unde Kaili",
+ "uon": "Kulon",
+ "upi": "Umeda",
+ "upv": "Uripiv-Wala-Rano-Atchin",
+ "ur": "Urdu",
+ "ura": "Urarina",
+ "urb": "Urubú-Kaapor; Kaapor",
+ "urc": "Urningangg",
+ "ure": "Uru",
+ "urf": "Uradhi",
+ "urg": "Urigina",
+ "urh": "Urhobo",
+ "uri": "Urim",
+ "urj": "Uralic languages",
+ "urk": "Urak Lawoi'",
+ "url": "Urali",
+ "urm": "Urapmin",
+ "urn": "Uruangnirin",
+ "uro": "Ura (Papua New Guinea)",
+ "urp": "Uru-Pa-In",
+ "urr": "Lehalurup; Löyöp",
+ "urt": "Urat",
+ "uru": "Urumi",
+ "urv": "Uruava",
+ "urw": "Sop",
+ "urx": "Urimo",
+ "ury": "Orya",
+ "urz": "Uru-Eu-Wau-Wau",
+ "usa": "Usarufa",
+ "ush": "Ushojo",
+ "usi": "Usui",
+ "usk": "Usaghade",
+ "usp": "Uspanteco",
+ "uss": "us-Saare",
+ "usu": "Uya",
+ "uta": "Otank",
+ "ute": "Ute-Southern Paiute",
+ "uth": "ut-Hun",
+ "utp": "Amba (Solomon Islands)",
+ "utr": "Etulo",
+ "utu": "Utu",
+ "uum": "Urum",
+ "uur": "Ura (Vanuatu)",
+ "uuu": "U",
+ "uve": "West Uvean; Fagauvea",
+ "uvh": "Uri",
+ "uvl": "Lote",
+ "uwa": "Kuku-Uwanh",
+ "uya": "Doko-Uyanga",
+ "uz": "Uzbek",
+ "uzn": "Northern Uzbek",
+ "uzs": "Southern Uzbek",
+ "vaa": "Vaagri Booli",
+ "vae": "Vale",
+ "vaf": "Vafsi",
+ "vag": "Vagla",
+ "vah": "Varhadi-Nagpuri",
+ "vai": "Vai",
+ "vaj": "Sekele; Northwestern ǃKung; Vasekele",
+ "val": "Vehes",
+ "vam": "Vanimo",
+ "van": "Valman",
+ "vao": "Vao",
+ "vap": "Vaiphei",
+ "var": "Huarijio",
+ "vas": "Vasavi",
+ "vau": "Vanuma",
+ "vav": "Varli",
+ "vay": "Wayu",
+ "vbb": "Southeast Babar",
+ "vbk": "Southwestern Bontok",
+ "ve": "Venda",
+ "vec": "Venetian",
+ "ved": "Veddah",
+ "vel": "Veluws",
+ "vem": "Vemgo-Mabas",
+ "veo": "Ventureño",
+ "vep": "Veps",
+ "ver": "Mom Jango",
+ "vgr": "Vaghri",
+ "vgt": "Vlaamse Gebarentaal; Flemish Sign Language",
+ "vi": "Vietnamese",
+ "vic": "Virgin Islands Creole English",
+ "vid": "Vidunda",
+ "vif": "Vili",
+ "vig": "Viemo",
+ "vil": "Vilela",
+ "vin": "Vinza",
+ "vis": "Vishavan",
+ "vit": "Viti",
+ "viv": "Iduna",
+ "vka": "Kariyarra",
+ "vkj": "Kujarge",
+ "vkk": "Kaur",
+ "vkl": "Kulisusu",
+ "vkm": "Kamakan",
+ "vkn": "Koro Nulu",
+ "vko": "Kodeoha",
+ "vkp": "Korlai Creole Portuguese",
+ "vkt": "Tenggarong Kutai Malay",
+ "vku": "Kurrama",
+ "vkz": "Koro Zuba",
+ "vlp": "Valpei",
+ "vls": "Vlaams",
+ "vma": "Martuyhunira",
+ "vmb": "Barbaram",
+ "vmc": "Juxtlahuaca Mixtec",
+ "vmd": "Mudu Koraga",
+ "vme": "East Masela",
+ "vmf": "Mainfränkisch",
+ "vmg": "Lungalunga",
+ "vmh": "Maraghei",
+ "vmi": "Miwa",
+ "vmj": "Ixtayutla Mixtec",
+ "vmk": "Makhuwa-Shirima",
+ "vml": "Malgana",
+ "vmm": "Mitlatongo Mixtec",
+ "vmp": "Soyaltepec Mazatec",
+ "vmq": "Soyaltepec Mixtec",
+ "vmr": "Marenje",
+ "vms": "Moksela",
+ "vmu": "Muluridyi",
+ "vmv": "Valley Maidu",
+ "vmw": "Makhuwa",
+ "vmx": "Tamazola Mixtec",
+ "vmy": "Ayautla Mazatec",
+ "vmz": "Mazatlán Mazatec",
+ "vnk": "Vano; Lovono",
+ "vnm": "Vinmavis; Neve'ei",
+ "vnp": "Vunapu",
+ "vo": "Volapük",
+ "vor": "Voro",
+ "vot": "Votic",
+ "vra": "Vera'a",
+ "vro": "Võro",
+ "vrs": "Varisi",
+ "vrt": "Burmbar; Banam Bay",
+ "vsi": "Moldova Sign Language",
+ "vsl": "Venezuelan Sign Language",
+ "vsv": "Valencian Sign Language; Llengua de signes valenciana",
+ "vto": "Vitou",
+ "vum": "Vumbu",
+ "vun": "Vunjo",
+ "vut": "Vute",
+ "vwa": "Awa (China)",
+ "wa": "Walloon",
+ "waa": "Walla Walla",
+ "wab": "Wab",
+ "wac": "Wasco-Wishram",
+ "wad": "Wamesa; Wondama",
+ "wae": "Walser",
+ "waf": "Wakoná",
+ "wag": "Wa'ema",
+ "wah": "Watubela",
+ "wai": "Wares",
+ "waj": "Waffa",
+ "wak": "Wakashan languages",
+ "wal": "Wolaytta; Wolaitta",
+ "wam": "Wampanoag",
+ "wan": "Wan",
+ "wao": "Wappo",
+ "wap": "Wapishana",
+ "waq": "Wagiman",
+ "war": "Waray (Philippines)",
+ "was": "Washo",
+ "wat": "Kaninuwa",
+ "wau": "Waurá",
+ "wav": "Waka",
+ "waw": "Waiwai",
+ "wax": "Watam; Marangis",
+ "way": "Wayana",
+ "waz": "Wampur",
+ "wba": "Warao",
+ "wbb": "Wabo",
+ "wbe": "Waritai",
+ "wbf": "Wara",
+ "wbh": "Wanda",
+ "wbi": "Vwanji",
+ "wbj": "Alagwa",
+ "wbk": "Waigali",
+ "wbl": "Wakhi",
+ "wbm": "Wa",
+ "wbp": "Warlpiri",
+ "wbq": "Waddar",
+ "wbr": "Wagdi",
+ "wbs": "West Bengal Sign Language",
+ "wbt": "Warnman",
+ "wbv": "Wajarri",
+ "wbw": "Woi",
+ "wca": "Yanomámi",
+ "wci": "Waci Gbe",
+ "wdd": "Wandji",
+ "wdg": "Wadaginam",
+ "wdj": "Wadjiginy",
+ "wdk": "Wadikali",
+ "wdt": "Wendat",
+ "wdu": "Wadjigu",
+ "wdy": "Wadjabangayi",
+ "wea": "Wewaw",
+ "wec": "Wè Western",
+ "wed": "Wedau",
+ "weg": "Wergaia",
+ "weh": "Weh",
+ "wei": "Kiunum",
+ "wem": "Weme Gbe",
+ "wen": "Sorbian languages",
+ "weo": "Wemale",
+ "wep": "Westphalien",
+ "wer": "Weri",
+ "wes": "Cameroon Pidgin",
+ "wet": "Perai",
+ "weu": "Rawngtu Chin",
+ "wew": "Wejewa",
+ "wfg": "Yafi; Zorop",
+ "wga": "Wagaya",
+ "wgb": "Wagawaga",
+ "wgg": "Wangkangurru; Wangganguru",
+ "wgi": "Wahgi",
+ "wgo": "Waigeo",
+ "wgu": "Wirangu",
+ "wgy": "Warrgamay",
+ "wha": "Sou Upaa; Manusela",
+ "whg": "North Wahgi",
+ "whk": "Wahau Kenyah",
+ "whu": "Wahau Kayan",
+ "wib": "Southern Toussian",
+ "wic": "Wichita",
+ "wie": "Wik-Epa",
+ "wif": "Wik-Keyangan",
+ "wig": "Wik Ngathan",
+ "wih": "Wik-Me'anha",
+ "wii": "Minidien",
+ "wij": "Wik-Iiyanh",
+ "wik": "Wikalkan",
+ "wil": "Wilawila",
+ "wim": "Wik-Mungkan",
+ "win": "Ho-Chunk",
+ "wir": "Wiraféd",
+ "wiu": "Wiru",
+ "wiv": "Vitu",
+ "wiy": "Wiyot",
+ "wja": "Waja",
+ "wji": "Warji",
+ "wka": "Kw'adza",
+ "wkb": "Kumbaran",
+ "wkd": "Wakde; Mo",
+ "wkl": "Kalanadi",
+ "wkr": "Keerray-Woorroong",
+ "wku": "Kunduvadi",
+ "wkw": "Wakawaka",
+ "wky": "Wangkayutyuru",
+ "wla": "Walio",
+ "wlc": "Mwali Comorian",
+ "wle": "Wolane",
+ "wlg": "Kunbarlang",
+ "wlh": "Welaun",
+ "wli": "Waioli",
+ "wlk": "Wailaki",
+ "wll": "Wali (Sudan)",
+ "wlm": "Middle Welsh",
+ "wlo": "Wolio",
+ "wlr": "Wailapa",
+ "wls": "Wallisian",
+ "wlu": "Wuliwuli",
+ "wlv": "Wichí Lhamtés Vejoz",
+ "wlw": "Walak",
+ "wlx": "Wali (Ghana)",
+ "wly": "Waling",
+ "wma": "Mawa (Nigeria)",
+ "wmb": "Wambaya",
+ "wmc": "Wamas",
+ "wmd": "Mamaindé",
+ "wme": "Wambule",
+ "wmg": "Western Minyag",
+ "wmh": "Waima'a",
+ "wmi": "Wamin",
+ "wmm": "Maiwa (Indonesia)",
+ "wmn": "Waamwang",
+ "wmo": "Wom (Papua New Guinea)",
+ "wms": "Wambon",
+ "wmt": "Walmajarri",
+ "wmw": "Mwani",
+ "wmx": "Womo",
+ "wnb": "Wanambre",
+ "wnc": "Wantoat",
+ "wnd": "Wandarang",
+ "wne": "Waneci",
+ "wng": "Wanggom",
+ "wni": "Ndzwani Comorian",
+ "wnk": "Wanukaka",
+ "wnm": "Wanggamala",
+ "wnn": "Wunumara",
+ "wno": "Wano",
+ "wnp": "Wanap",
+ "wnu": "Usan",
+ "wnw": "Wintu",
+ "wny": "Wanyi; Waanyi",
+ "wo": "Wolof",
+ "woa": "Kuwema; Tyaraity",
+ "wob": "Wè Northern",
+ "woc": "Wogeo",
+ "wod": "Wolani",
+ "woe": "Woleaian",
+ "wof": "Gambian Wolof",
+ "wog": "Wogamusin",
+ "woi": "Kamang",
+ "wok": "Longto",
+ "wom": "Wom (Nigeria)",
+ "won": "Wongo",
+ "woo": "Manombai",
+ "wor": "Woria",
+ "wos": "Hanga Hundi",
+ "wow": "Wawonii",
+ "woy": "Weyto",
+ "wpc": "Maco",
+ "wrb": "Waluwarra; Warluwara",
+ "wrg": "Warungu; Gudjal",
+ "wrh": "Wiradjuri",
+ "wri": "Wariyangga",
+ "wrk": "Garrwa",
+ "wrl": "Warlmanpa",
+ "wrm": "Warumungu",
+ "wrn": "Warnang",
+ "wro": "Worrorra",
+ "wrp": "Waropen",
+ "wrr": "Wardaman",
+ "wrs": "Waris",
+ "wru": "Waru",
+ "wrv": "Waruna",
+ "wrw": "Gugu Warra",
+ "wrx": "Wae Rana",
+ "wry": "Merwari",
+ "wrz": "Waray (Australia)",
+ "wsa": "Warembori",
+ "wsg": "Adilabad Gondi",
+ "wsi": "Wusi",
+ "wsk": "Waskia",
+ "wsr": "Owenia",
+ "wss": "Wasa",
+ "wsu": "Wasu",
+ "wsv": "Wotapuri-Katarqalai",
+ "wtf": "Watiwa",
+ "wth": "Wathawurrung",
+ "wti": "Berta",
+ "wtk": "Watakataui",
+ "wtm": "Mewati",
+ "wtw": "Wotu",
+ "wua": "Wikngenchera",
+ "wub": "Wunambal",
+ "wud": "Wudu",
+ "wuh": "Wutunhua",
+ "wul": "Silimo",
+ "wum": "Wumbvu",
+ "wun": "Bungu",
+ "wur": "Wurrugu",
+ "wut": "Wutung",
+ "wuu": "Wu Chinese",
+ "wuv": "Wuvulu-Aua",
+ "wux": "Wulna",
+ "wuy": "Wauyai",
+ "wwa": "Waama",
+ "wwb": "Wakabunga",
+ "wwo": "Wetamut; Dorig",
+ "wwr": "Warrwa",
+ "www": "Wawa",
+ "wxa": "Waxianghua",
+ "wxw": "Wardandi",
+ "wyb": "Wangaaybuwan-Ngiyambaa",
+ "wyi": "Woiwurrung",
+ "wym": "Wymysorys",
+ "wyn": "Wyandot",
+ "wyr": "Wayoró",
+ "wyy": "Western Fijian",
+ "xaa": "Andalusian Arabic",
+ "xab": "Sambe",
+ "xac": "Kachari",
+ "xad": "Adai",
+ "xae": "Aequian",
+ "xag": "Aghwan",
+ "xai": "Kaimbé",
+ "xaj": "Ararandewára",
+ "xak": "Máku",
+ "xal": "Kalmyk; Oirat",
+ "xam": "ǀXam",
+ "xan": "Xamtanga",
+ "xao": "Khao",
+ "xap": "Apalachee",
+ "xaq": "Aquitanian",
+ "xar": "Karami",
+ "xas": "Kamas",
+ "xat": "Katawixi",
+ "xau": "Kauwera",
+ "xav": "Xavánte",
+ "xaw": "Kawaiisu",
+ "xay": "Kayan Mahakam",
+ "xbb": "Lower Burdekin",
+ "xbc": "Bactrian",
+ "xbd": "Bindal",
+ "xbe": "Bigambal",
+ "xbg": "Bunganditj",
+ "xbi": "Kombio",
+ "xbj": "Birrpayi",
+ "xbm": "Middle Breton",
+ "xbn": "Kenaboi",
+ "xbo": "Bolgarian",
+ "xbp": "Bibbulman",
+ "xbr": "Kambera",
+ "xbw": "Kambiwá",
+ "xby": "Batjala; Batyala",
+ "xcb": "Cumbric",
+ "xcc": "Camunic",
+ "xce": "Celtiberian",
+ "xcg": "Cisalpine Gaulish",
+ "xch": "Chemakum; Chimakum",
+ "xcl": "Classical Armenian",
+ "xcm": "Comecrudo",
+ "xcn": "Cotoname",
+ "xco": "Chorasmian",
+ "xcr": "Carian",
+ "xct": "Classical Tibetan",
+ "xcu": "Curonian",
+ "xcv": "Chuvantsy",
+ "xcw": "Coahuilteco",
+ "xcy": "Cayuse",
+ "xda": "Darkinyung",
+ "xdc": "Dacian",
+ "xdk": "Dharuk",
+ "xdm": "Edomite",
+ "xdo": "Kwandu",
+ "xdq": "Kaitag",
+ "xdy": "Malayic Dayak",
+ "xeb": "Eblan",
+ "xed": "Hdi",
+ "xeg": "ǁXegwi",
+ "xel": "Kelo",
+ "xem": "Kembayan",
+ "xep": "Epi-Olmec",
+ "xer": "Xerénte",
+ "xes": "Kesawai",
+ "xet": "Xetá",
+ "xeu": "Keoru-Ahia",
+ "xfa": "Faliscan",
+ "xga": "Galatian",
+ "xgb": "Gbin",
+ "xgd": "Gudang",
+ "xgf": "Gabrielino-Fernandeño",
+ "xgg": "Goreng",
+ "xgi": "Garingbal",
+ "xgl": "Galindan",
+ "xgm": "Dharumbal; Guwinmal",
+ "xgn": "Mongolian languages",
+ "xgr": "Garza",
+ "xgu": "Unggumi",
+ "xgw": "Guwa",
+ "xh": "Xhosa",
+ "xha": "Harami",
+ "xhc": "Hunnic",
+ "xhd": "Hadrami",
+ "xhe": "Khetrani",
+ "xhm": "Middle Khmer (1400 to 1850 CE)",
+ "xhr": "Hernican",
+ "xht": "Hattic",
+ "xhu": "Hurrian",
+ "xhv": "Khua",
+ "xib": "Iberian",
+ "xii": "Xiri",
+ "xil": "Illyrian",
+ "xin": "Xinca",
+ "xir": "Xiriâna",
+ "xis": "Kisan",
+ "xiv": "Indus Valley Language",
+ "xiy": "Xipaya",
+ "xjb": "Minjungbal",
+ "xjt": "Jaitmatang",
+ "xka": "Kalkoti",
+ "xkb": "Northern Nago",
+ "xkc": "Kho'ini",
+ "xkd": "Mendalam Kayan",
+ "xke": "Kereho",
+ "xkf": "Khengkha",
+ "xkg": "Kagoro",
+ "xki": "Kenyan Sign Language",
+ "xkj": "Kajali",
+ "xkk": "Kachok; Kaco'",
+ "xkl": "Mainstream Kenyah",
+ "xkn": "Kayan River Kayan",
+ "xko": "Kiorr",
+ "xkp": "Kabatei",
+ "xkq": "Koroni",
+ "xkr": "Xakriabá",
+ "xks": "Kumbewaha",
+ "xkt": "Kantosi",
+ "xku": "Kaamba",
+ "xkv": "Kgalagadi",
+ "xkw": "Kembra",
+ "xkx": "Karore",
+ "xky": "Uma' Lasan",
+ "xkz": "Kurtokha",
+ "xla": "Kamula",
+ "xlb": "Loup B",
+ "xlc": "Lycian",
+ "xld": "Lydian",
+ "xle": "Lemnian",
+ "xlg": "Ligurian (Ancient)",
+ "xli": "Liburnian",
+ "xln": "Alanic",
+ "xlo": "Loup A",
+ "xlp": "Lepontic",
+ "xls": "Lusitanian",
+ "xlu": "Cuneiform Luwian",
+ "xly": "Elymian",
+ "xma": "Mushungulu",
+ "xmb": "Mbonga",
+ "xmc": "Makhuwa-Marrevone",
+ "xmd": "Mbudum",
+ "xme": "Median",
+ "xmf": "Mingrelian",
+ "xmg": "Mengaka",
+ "xmh": "Kugu-Muminh",
+ "xmj": "Majera",
+ "xmk": "Ancient Macedonian",
+ "xml": "Malaysian Sign Language",
+ "xmm": "Manado Malay",
+ "xmn": "Manichaean Middle Persian",
+ "xmo": "Morerebi",
+ "xmp": "Kuku-Mu'inh",
+ "xmq": "Kuku-Mangk",
+ "xmr": "Meroitic",
+ "xms": "Moroccan Sign Language",
+ "xmt": "Matbat",
+ "xmu": "Kamu",
+ "xmv": "Antankarana Malagasy; Tankarana Malagasy",
+ "xmw": "Tsimihety Malagasy",
+ "xmx": "Salawati; Maden",
+ "xmy": "Mayaguduna",
+ "xmz": "Mori Bawah",
+ "xna": "Ancient North Arabian",
+ "xnb": "Kanakanabu",
+ "xnd": "Na-Dene languages",
+ "xng": "Middle Mongolian",
+ "xnh": "Kuanhua",
+ "xni": "Ngarigu",
+ "xnj": "Ngoni (Tanzania)",
+ "xnk": "Nganakarti",
+ "xnm": "Ngumbarl",
+ "xnn": "Northern Kankanay",
+ "xno": "Anglo-Norman",
+ "xnq": "Ngoni (Mozambique)",
+ "xnr": "Kangri",
+ "xns": "Kanashi",
+ "xnt": "Narragansett",
+ "xnu": "Nukunul",
+ "xny": "Nyiyaparli",
+ "xnz": "Kenzi; Mattoki",
+ "xoc": "O'chi'chi'",
+ "xod": "Kokoda",
+ "xog": "Soga",
+ "xoi": "Kominimung",
+ "xok": "Xokleng",
+ "xom": "Komo (Sudan)",
+ "xon": "Konkomba",
+ "xoo": "Xukurú",
+ "xop": "Kopar",
+ "xor": "Korubo",
+ "xow": "Kowaki",
+ "xpa": "Pirriya",
+ "xpb": "Northeastern Tasmanian; Pyemmairrener",
+ "xpc": "Pecheneg",
+ "xpd": "Oyster Bay Tasmanian",
+ "xpe": "Liberia Kpelle",
+ "xpf": "Southeast Tasmanian; Nuenonne",
+ "xpg": "Phrygian",
+ "xph": "North Midlands Tasmanian; Tyerrenoterpanner",
+ "xpi": "Pictish",
+ "xpj": "Mpalitjanh",
+ "xpk": "Kulina Pano",
+ "xpl": "Port Sorell Tasmanian",
+ "xpm": "Pumpokol",
+ "xpn": "Kapinawá",
+ "xpo": "Pochutec",
+ "xpp": "Puyo-Paekche",
+ "xpq": "Mohegan-Pequot",
+ "xpr": "Parthian",
+ "xps": "Pisidian",
+ "xpt": "Punthamara",
+ "xpu": "Punic",
+ "xpv": "Northern Tasmanian; Tommeginne",
+ "xpw": "Northwestern Tasmanian; Peerapper",
+ "xpx": "Southwestern Tasmanian; Toogee",
+ "xpy": "Puyo",
+ "xpz": "Bruny Island Tasmanian",
+ "xqa": "Karakhanid",
+ "xqt": "Qatabanian",
+ "xra": "Krahô",
+ "xrb": "Eastern Karaboro",
+ "xrd": "Gundungurra",
+ "xre": "Kreye",
+ "xrg": "Minang",
+ "xri": "Krikati-Timbira",
+ "xrm": "Armazic",
+ "xrn": "Arin",
+ "xrr": "Raetic",
+ "xrt": "Aranama-Tamique",
+ "xru": "Marriammu",
+ "xrw": "Karawa",
+ "xsa": "Sabaean",
+ "xsb": "Sambal",
+ "xsc": "Scythian",
+ "xsd": "Sidetic",
+ "xse": "Sempan",
+ "xsh": "Shamang",
+ "xsi": "Sio",
+ "xsj": "Subi",
+ "xsl": "South Slavey",
+ "xsm": "Kasem",
+ "xsn": "Sanga (Nigeria)",
+ "xso": "Solano",
+ "xsp": "Silopi",
+ "xsq": "Makhuwa-Saka",
+ "xsr": "Sherpa",
+ "xss": "Assan",
+ "xsu": "Sanumá",
+ "xsv": "Sudovian",
+ "xsy": "Saisiyat",
+ "xta": "Alcozauca Mixtec",
+ "xtb": "Chazumba Mixtec",
+ "xtc": "Katcha-Kadugli-Miri",
+ "xtd": "Diuxi-Tilantongo Mixtec",
+ "xte": "Ketengban",
+ "xtg": "Transalpine Gaulish",
+ "xth": "Yitha Yitha",
+ "xti": "Sinicahua Mixtec",
+ "xtj": "San Juan Teita Mixtec",
+ "xtl": "Tijaltepec Mixtec",
+ "xtm": "Magdalena Peñasco Mixtec",
+ "xtn": "Northern Tlaxiaco Mixtec",
+ "xto": "Tokharian A",
+ "xtp": "San Miguel Piedras Mixtec",
+ "xtq": "Tumshuqese",
+ "xtr": "Early Tripuri",
+ "xts": "Sindihui Mixtec",
+ "xtt": "Tacahua Mixtec",
+ "xtu": "Cuyamecalco Mixtec",
+ "xtv": "Thawa",
+ "xtw": "Tawandê",
+ "xty": "Yoloxochitl Mixtec",
+ "xua": "Alu Kurumba",
+ "xub": "Betta Kurumba",
+ "xud": "Umiida",
+ "xug": "Kunigami",
+ "xuj": "Jennu Kurumba",
+ "xul": "Ngunawal; Nunukul",
+ "xum": "Umbrian",
+ "xun": "Unggaranggu",
+ "xuo": "Kuo",
+ "xup": "Upper Umpqua",
+ "xur": "Urartian",
+ "xut": "Kuthant",
+ "xuu": "Kxoe; Khwedam",
+ "xve": "Venetic",
+ "xvi": "Kamviri",
+ "xvn": "Vandalic",
+ "xvo": "Volscian",
+ "xvs": "Vestinian",
+ "xwa": "Kwaza",
+ "xwc": "Woccon",
+ "xwd": "Wadi Wadi",
+ "xwe": "Xwela Gbe",
+ "xwg": "Kwegu",
+ "xwj": "Wajuk",
+ "xwk": "Wangkumara",
+ "xwl": "Western Xwla Gbe",
+ "xwo": "Written Oirat",
+ "xwr": "Kwerba Mamberamo",
+ "xwt": "Wotjobaluk",
+ "xww": "Wemba Wemba",
+ "xxb": "Boro (Ghana)",
+ "xxk": "Ke'o",
+ "xxm": "Minkin",
+ "xxr": "Koropó",
+ "xxt": "Tambora",
+ "xya": "Yaygir",
+ "xyb": "Yandjibara",
+ "xyj": "Mayi-Yapi",
+ "xyk": "Mayi-Kulan",
+ "xyl": "Yalakalore",
+ "xyt": "Mayi-Thakurti",
+ "xyy": "Yorta Yorta",
+ "xzh": "Zhang-Zhung",
+ "xzm": "Zemgalian",
+ "xzp": "Ancient Zapotec",
+ "yaa": "Yaminahua",
+ "yab": "Yuhup",
+ "yac": "Pass Valley Yali",
+ "yad": "Yagua",
+ "yae": "Pumé",
+ "yaf": "Yaka (Democratic Republic of Congo)",
+ "yag": "Yámana",
+ "yah": "Yazgulyam",
+ "yai": "Yagnobi",
+ "yaj": "Banda-Yangere",
+ "yak": "Yakama",
+ "yal": "Yalunka",
+ "yam": "Yamba",
+ "yan": "Mayangna",
+ "yao": "Yao",
+ "yap": "Yapese",
+ "yaq": "Yaqui",
+ "yar": "Yabarana",
+ "yas": "Nugunu (Cameroon)",
+ "yat": "Yambeta",
+ "yau": "Yuwana",
+ "yav": "Yangben",
+ "yaw": "Yawalapití",
+ "yax": "Yauma",
+ "yay": "Agwagwune",
+ "yaz": "Lokaa",
+ "yba": "Yala",
+ "ybb": "Yemba",
+ "ybe": "West Yugur",
+ "ybh": "Yakha",
+ "ybi": "Yamphu",
+ "ybj": "Hasha",
+ "ybk": "Bokha",
+ "ybl": "Yukuben",
+ "ybm": "Yaben",
+ "ybn": "Yabaâna",
+ "ybo": "Yabong",
+ "ybx": "Yawiyo",
+ "yby": "Yaweyuha",
+ "ych": "Chesu",
+ "ycl": "Lolopo",
+ "ycn": "Yucuna",
+ "ycp": "Chepya",
+ "yda": "Yanda",
+ "ydd": "Eastern Yiddish",
+ "yde": "Yangum Dey",
+ "ydg": "Yidgha",
+ "ydk": "Yoidik",
+ "yea": "Ravula",
+ "yec": "Yeniche",
+ "yee": "Yimas",
+ "yei": "Yeni",
+ "yej": "Yevanic",
+ "yel": "Yela",
+ "yer": "Tarok",
+ "yes": "Nyankpa",
+ "yet": "Yetfa",
+ "yeu": "Yerukula",
+ "yev": "Yapunda",
+ "yey": "Yeyi",
+ "yga": "Malyangapa",
+ "ygi": "Yiningayi",
+ "ygl": "Yangum Gel",
+ "ygm": "Yagomi",
+ "ygp": "Gepo",
+ "ygr": "Yagaria",
+ "ygs": "Yolŋu Sign Language",
+ "ygu": "Yugul",
+ "ygw": "Yagwoia",
+ "yha": "Baha Buyang",
+ "yhd": "Judeo-Iraqi Arabic",
+ "yhl": "Hlepho Phowa",
+ "yhs": "Yan-nhaŋu Sign Language",
+ "yi": "Yiddish",
+ "yia": "Yinggarda",
+ "yif": "Ache",
+ "yig": "Wusa Nasu",
+ "yih": "Western Yiddish",
+ "yii": "Yidiny",
+ "yij": "Yindjibarndi",
+ "yik": "Dongshanba Lalo",
+ "yil": "Yindjilandji",
+ "yim": "Yimchungru Naga",
+ "yin": "Riang Lai; Yinchia",
+ "yip": "Pholo",
+ "yiq": "Miqie",
+ "yir": "North Awyu",
+ "yis": "Yis",
+ "yit": "Eastern Lalu",
+ "yiu": "Awu",
+ "yiv": "Northern Nisu",
+ "yix": "Axi Yi",
+ "yiz": "Azhe",
+ "yka": "Yakan",
+ "ykg": "Northern Yukaghir",
+ "yki": "Yoke",
+ "ykk": "Yakaikeke",
+ "ykl": "Khlula",
+ "ykm": "Kap",
+ "ykn": "Kua-nsi",
+ "yko": "Yasa",
+ "ykr": "Yekora",
+ "ykt": "Kathu",
+ "yku": "Kuamasi",
+ "yky": "Yakoma",
+ "yla": "Yaul",
+ "ylb": "Yaleba",
+ "yle": "Yele",
+ "ylg": "Yelogu",
+ "yli": "Angguruk Yali",
+ "yll": "Yil",
+ "ylm": "Limi",
+ "yln": "Langnian Buyang",
+ "ylo": "Naluo Yi",
+ "ylr": "Yalarnnga",
+ "ylu": "Aribwaung",
+ "yly": "Nyâlayu; Nyelâyu",
+ "ymb": "Yambes",
+ "ymc": "Southern Muji",
+ "ymd": "Muda",
+ "yme": "Yameo",
+ "ymg": "Yamongeri",
+ "ymh": "Mili",
+ "ymi": "Moji",
+ "ymk": "Makwe",
+ "yml": "Iamalele",
+ "ymm": "Maay",
+ "ymn": "Yamna; Sunum",
+ "ymo": "Yangum Mon",
+ "ymp": "Yamap",
+ "ymq": "Qila Muji",
+ "ymr": "Malasar",
+ "yms": "Mysian",
+ "ymx": "Northern Muji",
+ "ymz": "Muzi",
+ "yna": "Aluo",
+ "ynd": "Yandruwandha",
+ "yne": "Lang'e",
+ "yng": "Yango",
+ "ynk": "Naukan Yupik",
+ "ynl": "Yangulam",
+ "ynn": "Yana",
+ "yno": "Yong",
+ "ynq": "Yendang",
+ "yns": "Yansi",
+ "ynu": "Yahuna",
+ "yo": "Yoruba",
+ "yob": "Yoba",
+ "yog": "Yogad",
+ "yoi": "Yonaguni",
+ "yok": "Yokuts",
+ "yol": "Yola",
+ "yom": "Yombe",
+ "yon": "Yongkom",
+ "yot": "Yotti",
+ "yox": "Yoron",
+ "yoy": "Yoy",
+ "ypa": "Phala",
+ "ypb": "Labo Phowa",
+ "ypg": "Phola",
+ "yph": "Phupha",
+ "ypk": "Yupik languages",
+ "ypm": "Phuma",
+ "ypn": "Ani Phowa",
+ "ypo": "Alo Phola",
+ "ypp": "Phupa",
+ "ypz": "Phuza",
+ "yra": "Yerakai",
+ "yrb": "Yareba",
+ "yre": "Yaouré",
+ "yrk": "Nenets",
+ "yrl": "Nhengatu",
+ "yrm": "Yirrk-Mel",
+ "yrn": "Yerong",
+ "yro": "Yaroamë",
+ "yrs": "Yarsun",
+ "yrw": "Yarawata",
+ "yry": "Yarluyandi",
+ "ysc": "Yassic",
+ "ysd": "Samatao",
+ "ysg": "Sonaga",
+ "ysl": "Yugoslavian Sign Language",
+ "ysm": "Myanmar Sign Language",
+ "ysn": "Sani",
+ "yso": "Nisi (China)",
+ "ysp": "Southern Lolopo",
+ "ysr": "Sirenik Yupik",
+ "yss": "Yessan-Mayo",
+ "ysy": "Sanie",
+ "yta": "Talu",
+ "ytl": "Tanglang",
+ "ytp": "Thopho",
+ "ytw": "Yout Wam",
+ "yty": "Yatay",
+ "yua": "Yucateco; Yucatec Maya",
+ "yub": "Yugambal",
+ "yuc": "Yuchi",
+ "yud": "Judeo-Tripolitanian Arabic",
+ "yue": "Yue Chinese; Cantonese",
+ "yuf": "Havasupai-Walapai-Yavapai",
+ "yug": "Yug",
+ "yui": "Yurutí",
+ "yuj": "Karkar-Yuri",
+ "yuk": "Yuki",
+ "yul": "Yulu",
+ "yum": "Quechan",
+ "yun": "Bena (Nigeria)",
+ "yup": "Yukpa",
+ "yuq": "Yuqui",
+ "yur": "Yurok",
+ "yut": "Yopno",
+ "yuw": "Yau (Morobe Province)",
+ "yux": "Southern Yukaghir",
+ "yuy": "East Yugur",
+ "yuz": "Yuracare",
+ "yva": "Yawa",
+ "yvt": "Yavitero",
+ "ywa": "Kalou",
+ "ywg": "Yinhawangka",
+ "ywl": "Western Lalu",
+ "ywn": "Yawanawa",
+ "ywq": "Wuding-Luquan Yi",
+ "ywr": "Yawuru",
+ "ywt": "Xishanba Lalo; Central Lalo",
+ "ywu": "Wumeng Nasu",
+ "yww": "Yawarawarga",
+ "yxa": "Mayawali",
+ "yxg": "Yagara",
+ "yxl": "Yardliyawarra",
+ "yxm": "Yinwum",
+ "yxu": "Yuyu",
+ "yxy": "Yabula Yabula",
+ "yyr": "Yir Yoront",
+ "yyu": "Yau (Sandaun Province)",
+ "yyz": "Ayizi",
+ "yzg": "E'ma Buyang",
+ "yzk": "Zokhuo",
+ "za": "Zhuang; Chuang",
+ "zaa": "Sierra de Juárez Zapotec",
+ "zab": "Western Tlacolula Valley Zapotec; San Juan Guelavía Zapotec",
+ "zac": "Ocotlán Zapotec",
+ "zad": "Cajonos Zapotec",
+ "zae": "Yareni Zapotec",
+ "zaf": "Ayoquesco Zapotec",
+ "zag": "Zaghawa",
+ "zah": "Zangwal",
+ "zai": "Isthmus Zapotec",
+ "zaj": "Zaramo",
+ "zak": "Zanaki",
+ "zal": "Zauzou",
+ "zam": "Miahuatlán Zapotec",
+ "zao": "Ozolotepec Zapotec",
+ "zap": "Zapotec",
+ "zaq": "Aloápam Zapotec",
+ "zar": "Rincón Zapotec",
+ "zas": "Santo Domingo Albarradas Zapotec",
+ "zat": "Tabaa Zapotec",
+ "zau": "Zangskari",
+ "zav": "Yatzachi Zapotec",
+ "zaw": "Mitla Zapotec",
+ "zax": "Xadani Zapotec",
+ "zay": "Zayse-Zergulla; Zaysete",
+ "zaz": "Zari",
+ "zba": "Balaibalan",
+ "zbc": "Central Berawan",
+ "zbe": "East Berawan",
+ "zbl": "Blissymbols; Bliss; Blissymbolics",
+ "zbt": "Batui",
+ "zbu": "Bu (Bauchi State)",
+ "zbw": "West Berawan",
+ "zca": "Coatecas Altas Zapotec",
+ "zcd": "Las Delicias Zapotec",
+ "zch": "Central Hongshuihe Zhuang",
+ "zdj": "Ngazidja Comorian",
+ "zea": "Zeeuws",
+ "zeg": "Zenag",
+ "zeh": "Eastern Hongshuihe Zhuang",
+ "zen": "Zenaga",
+ "zga": "Kinga",
+ "zgb": "Guibei Zhuang",
+ "zgh": "Standard Moroccan Tamazight",
+ "zgm": "Minz Zhuang",
+ "zgn": "Guibian Zhuang",
+ "zgr": "Magori",
+ "zh": "Chinese",
+ "zhb": "Zhaba",
+ "zhd": "Dai Zhuang",
+ "zhi": "Zhire",
+ "zhn": "Nong Zhuang",
+ "zhw": "Zhoa",
+ "zhx": "Chinese (family)",
+ "zia": "Zia",
+ "zib": "Zimbabwe Sign Language",
+ "zik": "Zimakani",
+ "zil": "Zialo",
+ "zim": "Mesme",
+ "zin": "Zinza",
+ "ziw": "Zigula",
+ "ziz": "Zizilivakan",
+ "zka": "Kaimbulawa",
+ "zkb": "Koibal",
+ "zkd": "Kadu",
+ "zkg": "Koguryo",
+ "zkh": "Khorezmian",
+ "zkk": "Karankawa",
+ "zkn": "Kanan",
+ "zko": "Kott",
+ "zkp": "São Paulo Kaingáng",
+ "zkr": "Zakhring",
+ "zkt": "Kitan",
+ "zku": "Kaurna",
+ "zkv": "Krevinian",
+ "zkz": "Khazar",
+ "zla": "Zula",
+ "zle": "East Slavic languages",
+ "zlj": "Liujiang Zhuang",
+ "zlm": "Malay (individual language)",
+ "zln": "Lianshan Zhuang",
+ "zlq": "Liuqian Zhuang",
+ "zls": "South Slavic languages",
+ "zlw": "West Slavic languages",
+ "zma": "Manda (Australia)",
+ "zmb": "Zimba",
+ "zmc": "Margany",
+ "zmd": "Maridan",
+ "zme": "Mangerr",
+ "zmf": "Mfinu",
+ "zmg": "Marti Ke",
+ "zmh": "Makolkol",
+ "zmi": "Negeri Sembilan Malay",
+ "zmj": "Maridjabin",
+ "zmk": "Mandandanyi",
+ "zml": "Matngala",
+ "zmm": "Marimanindji; Marramaninyshi",
+ "zmn": "Mbangwe",
+ "zmo": "Molo",
+ "zmp": "Mpuono",
+ "zmq": "Mituku",
+ "zmr": "Maranunggu",
+ "zms": "Mbesa",
+ "zmt": "Maringarr",
+ "zmu": "Muruwari",
+ "zmv": "Mbariman-Gudhinma",
+ "zmw": "Mbo (Democratic Republic of Congo)",
+ "zmx": "Bomitaba",
+ "zmy": "Mariyedi",
+ "zmz": "Mbandja",
+ "zna": "Zan Gula",
+ "znd": "Zande languages",
+ "zne": "Zande (individual language)",
+ "zng": "Mang",
+ "znk": "Manangkari",
+ "zns": "Mangas",
+ "zoc": "Copainalá Zoque",
+ "zoh": "Chimalapa Zoque",
+ "zom": "Zou",
+ "zoo": "Asunción Mixtepec Zapotec",
+ "zoq": "Tabasco Zoque",
+ "zor": "Rayón Zoque",
+ "zos": "Francisco León Zoque",
+ "zpa": "Lachiguiri Zapotec",
+ "zpb": "Yautepec Zapotec",
+ "zpc": "Choapan Zapotec",
+ "zpd": "Southeastern Ixtlán Zapotec",
+ "zpe": "Petapa Zapotec",
+ "zpf": "San Pedro Quiatoni Zapotec",
+ "zpg": "Guevea De Humboldt Zapotec",
+ "zph": "Totomachapan Zapotec",
+ "zpi": "Santa María Quiegolani Zapotec",
+ "zpj": "Quiavicuzas Zapotec",
+ "zpk": "Tlacolulita Zapotec",
+ "zpl": "Lachixío Zapotec",
+ "zpm": "Mixtepec Zapotec",
+ "zpn": "Santa Inés Yatzechi Zapotec",
+ "zpo": "Amatlán Zapotec",
+ "zpp": "El Alto Zapotec",
+ "zpq": "Zoogocho Zapotec",
+ "zpr": "Santiago Xanica Zapotec",
+ "zps": "Coatlán Zapotec",
+ "zpt": "San Vicente Coatlán Zapotec",
+ "zpu": "Yalálag Zapotec",
+ "zpv": "Chichicapan Zapotec",
+ "zpw": "Zaniza Zapotec",
+ "zpx": "San Baltazar Loxicha Zapotec",
+ "zpy": "Mazaltepec Zapotec",
+ "zpz": "Texmelucan Zapotec",
+ "zqe": "Qiubei Zhuang",
+ "zra": "Kara (Korea)",
+ "zrg": "Mirgan",
+ "zrn": "Zerenkel",
+ "zro": "Záparo",
+ "zrp": "Zarphatic",
+ "zrs": "Mairasi",
+ "zsa": "Sarasira",
+ "zsk": "Kaskean",
+ "zsl": "Zambian Sign Language",
+ "zsm": "Standard Malay",
+ "zsr": "Southern Rincon Zapotec",
+ "zsu": "Sukurum",
+ "zte": "Elotepec Zapotec",
+ "ztg": "Xanaguía Zapotec",
+ "ztl": "Lapaguía-Guivini Zapotec",
+ "ztm": "San Agustín Mixtepec Zapotec",
+ "ztn": "Santa Catarina Albarradas Zapotec",
+ "ztp": "Loxicha Zapotec",
+ "ztq": "Quioquitani-Quierí Zapotec",
+ "zts": "Tilquiapan Zapotec",
+ "ztt": "Tejalapan Zapotec",
+ "ztu": "Güilá Zapotec",
+ "ztx": "Zaachila Zapotec",
+ "zty": "Yatee Zapotec",
+ "zu": "Zulu",
+ "zua": "Zeem",
+ "zuh": "Tokano",
+ "zum": "Kumzari",
+ "zun": "Zuni",
+ "zuy": "Zumaya",
+ "zwa": "Zay",
+ "zyb": "Yongbei Zhuang",
+ "zyg": "Yang Zhuang",
+ "zyj": "Youjiang Zhuang",
+ "zyn": "Yongnan Zhuang",
+ "zyp": "Zyphe Chin",
+ "zza": "Zaza; Dimili; Dimli (macrolanguage); Kirdki; Kirmanjki (macrolanguage); Zazaki",
+ "zzj": "Zuojiang Zhuang"
+}
\ No newline at end of file
diff --git a/venv/Lib/site-packages/datasets/utils/resources/multilingualities.json b/venv/Lib/site-packages/datasets/utils/resources/multilingualities.json
new file mode 100644
index 0000000000000000000000000000000000000000..a35c79f03dfcf7c8a116b7fc8ee1b383ab5022fa
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/resources/multilingualities.json
@@ -0,0 +1,6 @@
+{
+ "monolingual": "contains a single language",
+ "multilingual": "contains multiple languages",
+ "translation": "contains translated or aligned text",
+ "other": "other type of language distribution"
+}
diff --git a/venv/Lib/site-packages/datasets/utils/resources/readme_structure.yaml b/venv/Lib/site-packages/datasets/utils/resources/readme_structure.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5b781e11e8258a446874ebf96104f642d0c190cf
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/resources/readme_structure.yaml
@@ -0,0 +1,116 @@
+name: "" # Filename comes here
+allow_empty: false
+allow_empty_text: true
+subsections:
+ - name: "Dataset Card for X" # First-level markdown heading
+ allow_empty: false
+ allow_empty_text: true
+ subsections:
+ - name: "Table of Contents"
+ allow_empty: false
+ allow_empty_text: false
+ subsections: null # meaning it should not be checked.
+ - name: "Dataset Description"
+ allow_empty: false
+ allow_empty_text: false
+ subsections:
+ - name: "Dataset Summary"
+ allow_empty: false
+ allow_empty_text: false
+ subsections: null
+ - name: "Supported Tasks and Leaderboards"
+ allow_empty: true
+ allow_empty_text: true
+ subsections: null
+ - name: Languages
+ allow_empty: true
+ allow_empty_text: true
+ subsections: null
+ - name: "Dataset Structure"
+ allow_empty: false
+ allow_empty_text: true
+ subsections:
+ - name: "Data Instances"
+ allow_empty: false
+ allow_empty_text: true
+ subsections: null
+ - name: "Data Fields"
+ allow_empty: false
+ allow_empty_text: true
+ subsections: null
+ - name: "Data Splits"
+ allow_empty: false
+ allow_empty_text: true
+ subsections: null
+ - name: "Dataset Creation"
+ allow_empty: false
+ allow_empty_text: true
+ subsections:
+ - name: "Curation Rationale"
+ allow_empty: true
+ allow_empty_text: true
+ subsections: null
+ - name: "Source Data"
+ allow_empty: false
+ allow_empty_text: true
+ subsections:
+ - name: "Initial Data Collection and Normalization"
+ allow_empty: true
+ allow_empty_text: true
+ subsections: null
+ - name: "Who are the source language producers?"
+ allow_empty: true
+ allow_empty_text: true
+ subsections: null
+ - name: "Annotations"
+ allow_empty: false
+ allow_empty_text: true
+ subsections:
+ - name: "Annotation process"
+ allow_empty: true
+ allow_empty_text: true
+ subsections: null
+ - name: "Who are the annotators?"
+ allow_empty: true
+ allow_empty_text: true
+ subsections: null
+ - name: "Personal and Sensitive Information"
+ allow_empty: true
+ allow_empty_text: true
+ subsections: null
+ - name: "Considerations for Using the Data"
+ allow_empty: true
+ allow_empty_text: true
+ subsections:
+ - name: "Social Impact of Dataset"
+ allow_empty: true
+ allow_empty_text: true
+ subsections: null
+ - name: "Discussion of Biases"
+ allow_empty: true
+ allow_empty_text: true
+ subsections: null
+ - name: "Other Known Limitations"
+ allow_empty: true
+ allow_empty_text: true
+ subsections: null
+ - name: "Additional Information"
+ allow_empty: true
+ allow_empty_text: true
+ subsections:
+ - name: "Dataset Curators"
+ allow_empty: true
+ allow_empty_text: true
+ subsections: null
+ - name: "Licensing Information"
+ allow_empty: true
+ allow_empty_text: true
+ subsections: null
+ - name: "Citation Information"
+ allow_empty: false
+ allow_empty_text: true
+ subsections: null
+ - name: "Contributions"
+ allow_empty: false
+ allow_empty_text: false
+ subsections: null
diff --git a/venv/Lib/site-packages/datasets/utils/resources/size_categories.json b/venv/Lib/site-packages/datasets/utils/resources/size_categories.json
new file mode 100644
index 0000000000000000000000000000000000000000..983ce0c10dbb2e2245f90ae47e9de4c1025d5bb1
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/resources/size_categories.json
@@ -0,0 +1,14 @@
+[
+ "unknown",
+ "n<1K",
+ "1K1T"
+]
diff --git a/venv/Lib/site-packages/datasets/utils/sharding.py b/venv/Lib/site-packages/datasets/utils/sharding.py
new file mode 100644
index 0000000000000000000000000000000000000000..753bdf60c2a5b0b23f70893de5ba57a51462d126
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/sharding.py
@@ -0,0 +1,92 @@
+import numpy as np
+
+
+def _number_of_shards_in_gen_kwargs(gen_kwargs: dict) -> int:
+ """Return the number of possible shards according to the input gen_kwargs"""
+ # Having lists of different sizes makes sharding ambigious, raise an error in this case
+ # until we decide how to define sharding without ambiguity for users
+ lists_lengths = {key: len(value) for key, value in gen_kwargs.items() if isinstance(value, list)}
+ if len(set(lists_lengths.values())) > 1:
+ raise RuntimeError(
+ "Sharding is ambiguous for this dataset: "
+ + "we found several data sources lists of different lengths, and we don't know over which list we should parallelize:\n"
+ + "\n".join(f"\t- key {key} has length {length}" for key, length in lists_lengths.items())
+ + "\nTo fix this, check the 'gen_kwargs' and make sure to use lists only for data sources, "
+ + "and use tuples otherwise. In the end there should only be one single list, or several lists with the same length."
+ )
+ max_length = max(lists_lengths.values(), default=0)
+ return max(1, max_length)
+
+
+def _distribute_shards(num_shards: int, max_num_jobs: int) -> list[range]:
+ """
+ Get the range of shard indices per job.
+ If num_shards>> _distribute_shards(2, max_num_jobs=4)
+ [range(0, 1), range(1, 2)]
+ >>> _distribute_shards(10, max_num_jobs=3)
+ [range(0, 4), range(4, 7), range(7, 10)]
+ ```
+ """
+ shards_indices_per_group = []
+ for group_idx in range(max_num_jobs):
+ num_shards_to_add = num_shards // max_num_jobs + (group_idx < (num_shards % max_num_jobs))
+ if num_shards_to_add == 0:
+ break
+ start = shards_indices_per_group[-1].stop if shards_indices_per_group else 0
+ shard_indices = range(start, start + num_shards_to_add)
+ shards_indices_per_group.append(shard_indices)
+ return shards_indices_per_group
+
+
+def _split_gen_kwargs(gen_kwargs: dict, max_num_jobs: int) -> list[dict]:
+ """Split the gen_kwargs into `max_num_job` gen_kwargs"""
+ # Having lists of different sizes makes sharding ambigious, raise an error in this case
+ num_shards = _number_of_shards_in_gen_kwargs(gen_kwargs)
+ if num_shards == 1:
+ return [dict(gen_kwargs)]
+ else:
+ shard_indices_per_group = _distribute_shards(num_shards=num_shards, max_num_jobs=max_num_jobs)
+ return [
+ {
+ key: [value[shard_idx] for shard_idx in shard_indices_per_group[group_idx]]
+ if isinstance(value, list)
+ else value
+ for key, value in gen_kwargs.items()
+ }
+ for group_idx in range(len(shard_indices_per_group))
+ ]
+
+
+def _merge_gen_kwargs(gen_kwargs_list: list[dict]) -> dict:
+ return {
+ key: [value for gen_kwargs in gen_kwargs_list for value in gen_kwargs[key]]
+ if isinstance(gen_kwargs_list[0][key], list)
+ else gen_kwargs_list[0][key]
+ for key in gen_kwargs_list[0]
+ }
+
+
+def _shuffle_gen_kwargs(rng: np.random.Generator, gen_kwargs: dict) -> dict:
+ """Return a shuffled copy of the input gen_kwargs"""
+ # We must shuffle all the lists, and lists of the same size must have the same shuffling.
+ # This way entangled lists of (shard, shard_metadata) are still in the right order.
+
+ # First, let's generate the shuffled indices per list size
+ list_sizes = {len(value) for value in gen_kwargs.values() if isinstance(value, list)}
+ indices_per_size = {}
+ for size in list_sizes:
+ indices_per_size[size] = list(range(size))
+ rng.shuffle(indices_per_size[size])
+ # Now let's copy the gen_kwargs and shuffle the lists based on their sizes
+ shuffled_kwargs = dict(gen_kwargs)
+ for key, value in shuffled_kwargs.items():
+ if isinstance(value, list):
+ shuffled_kwargs[key] = [value[i] for i in indices_per_size[len(value)]]
+ return shuffled_kwargs
diff --git a/venv/Lib/site-packages/datasets/utils/stratify.py b/venv/Lib/site-packages/datasets/utils/stratify.py
new file mode 100644
index 0000000000000000000000000000000000000000..6dab0469e8168148a81eb2c13f7c103b98d061e7
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/stratify.py
@@ -0,0 +1,107 @@
+import numpy as np
+
+
+def approximate_mode(class_counts, n_draws, rng):
+ """Computes approximate mode of multivariate hypergeometric.
+ This is an approximation to the mode of the multivariate
+ hypergeometric given by class_counts and n_draws.
+ It shouldn't be off by more than one.
+ It is the mostly likely outcome of drawing n_draws many
+ samples from the population given by class_counts.
+ Args
+ ----------
+ class_counts : ndarray of int
+ Population per class.
+ n_draws : int
+ Number of draws (samples to draw) from the overall population.
+ rng : random state
+ Used to break ties.
+ Returns
+ -------
+ sampled_classes : ndarray of int
+ Number of samples drawn from each class.
+ np.sum(sampled_classes) == n_draws
+
+ """
+ # this computes a bad approximation to the mode of the
+ # multivariate hypergeometric given by class_counts and n_draws
+ continuous = n_draws * class_counts / class_counts.sum()
+ # floored means we don't overshoot n_samples, but probably undershoot
+ floored = np.floor(continuous)
+ # we add samples according to how much "left over" probability
+ # they had, until we arrive at n_samples
+ need_to_add = int(n_draws - floored.sum())
+ if need_to_add > 0:
+ remainder = continuous - floored
+ values = np.sort(np.unique(remainder))[::-1]
+ # add according to remainder, but break ties
+ # randomly to avoid biases
+ for value in values:
+ (inds,) = np.where(remainder == value)
+ # if we need_to_add less than what's in inds
+ # we draw randomly from them.
+ # if we need to add more, we add them all and
+ # go to the next value
+ add_now = min(len(inds), need_to_add)
+ inds = rng.choice(inds, size=add_now, replace=False)
+ floored[inds] += 1
+ need_to_add -= add_now
+ if need_to_add == 0:
+ break
+ return floored.astype(np.int64)
+
+
+def stratified_shuffle_split_generate_indices(y, n_train, n_test, rng, n_splits=10):
+ """
+
+ Provides train/test indices to split data in train/test sets.
+ It's reference is taken from StratifiedShuffleSplit implementation
+ of scikit-learn library.
+
+ Args
+ ----------
+
+ n_train : int,
+ represents the absolute number of train samples.
+
+ n_test : int,
+ represents the absolute number of test samples.
+
+ random_state : int or RandomState instance, default=None
+ Controls the randomness of the training and testing indices produced.
+ Pass an int for reproducible output across multiple function calls.
+
+ n_splits : int, default=10
+ Number of re-shuffling & splitting iterations.
+ """
+ classes, y_indices = np.unique(y, return_inverse=True)
+ n_classes = classes.shape[0]
+ class_counts = np.bincount(y_indices)
+ if np.min(class_counts) < 2:
+ raise ValueError("Minimum class count error")
+ if n_train < n_classes:
+ raise ValueError(
+ "The train_size = %d should be greater or equal to the number of classes = %d" % (n_train, n_classes)
+ )
+ if n_test < n_classes:
+ raise ValueError(
+ "The test_size = %d should be greater or equal to the number of classes = %d" % (n_test, n_classes)
+ )
+ class_indices = np.split(np.argsort(y_indices, kind="mergesort"), np.cumsum(class_counts)[:-1])
+ for _ in range(n_splits):
+ n_i = approximate_mode(class_counts, n_train, rng)
+ class_counts_remaining = class_counts - n_i
+ t_i = approximate_mode(class_counts_remaining, n_test, rng)
+
+ train = []
+ test = []
+
+ for i in range(n_classes):
+ permutation = rng.permutation(class_counts[i])
+ perm_indices_class_i = class_indices[i].take(permutation, mode="clip")
+ train.extend(perm_indices_class_i[: n_i[i]])
+ test.extend(perm_indices_class_i[n_i[i] : n_i[i] + t_i[i]])
+ train = rng.permutation(train)
+ test = rng.permutation(test)
+
+ yield train, test
diff --git a/venv/Lib/site-packages/datasets/utils/tf_utils.py b/venv/Lib/site-packages/datasets/utils/tf_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..10ae3ef64ae5e14339bf0e855c010253b92a7e02
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/tf_utils.py
@@ -0,0 +1,564 @@
+# Copyright 2022 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""TF-specific utils import."""
+
+import os
+import warnings
+from functools import partial
+from math import ceil
+from uuid import uuid4
+
+import numpy as np
+import pyarrow as pa
+from multiprocess import get_context
+
+
+try:
+ from multiprocess.shared_memory import SharedMemory
+except ImportError:
+ SharedMemory = None # Version checks should prevent this being called on older Python versions
+
+from .. import config
+
+
+def minimal_tf_collate_fn(features):
+ if isinstance(features, dict): # case batch_size=None: nothing to collate
+ return features
+ elif config.TF_AVAILABLE:
+ import tensorflow as tf
+ else:
+ raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.")
+
+ first = features[0]
+ batch = {}
+ for k, v in first.items():
+ if isinstance(v, np.ndarray):
+ batch[k] = np.stack([f[k] for f in features])
+ elif isinstance(v, tf.Tensor):
+ batch[k] = tf.stack([f[k] for f in features])
+ else:
+ batch[k] = np.array([f[k] for f in features])
+ return batch
+
+
+def minimal_tf_collate_fn_with_renaming(features):
+ batch = minimal_tf_collate_fn(features)
+ if "label" in batch:
+ batch["labels"] = batch["label"]
+ del batch["label"]
+ return batch
+
+
+def is_numeric_pa_type(pa_type):
+ if pa.types.is_list(pa_type):
+ return is_numeric_pa_type(pa_type.value_type)
+ return pa.types.is_integer(pa_type) or pa.types.is_floating(pa_type) or pa.types.is_decimal(pa_type)
+
+
+def np_get_batch(
+ indices, dataset, cols_to_retain, collate_fn, collate_fn_args, columns_to_np_types, return_dict=False
+):
+ if not isinstance(indices, np.ndarray):
+ indices = indices.numpy()
+
+ is_batched = True
+ # Optimization - if we're loading a sequential batch, do it with slicing instead of a list of indices
+ if isinstance(indices, np.integer):
+ batch = dataset[indices.item()]
+ is_batched = False
+ elif np.all(np.diff(indices) == 1):
+ batch = dataset[indices[0] : indices[-1] + 1]
+ elif isinstance(indices, np.ndarray):
+ batch = dataset[indices]
+ else:
+ raise RuntimeError(f"Unexpected type for indices: {type(indices)}")
+
+ if cols_to_retain is not None:
+ batch = {
+ key: value
+ for key, value in batch.items()
+ if key in cols_to_retain or key in ("label", "label_ids", "labels")
+ }
+
+ if is_batched:
+ actual_size = len(list(batch.values())[0]) # Get the length of one of the arrays, assume all same
+ # Our collators expect a list of dicts, not a dict of lists/arrays, so we invert
+ batch = [{key: value[i] for key, value in batch.items()} for i in range(actual_size)]
+ batch = collate_fn(batch, **collate_fn_args)
+
+ if return_dict:
+ out_batch = {}
+ for col, cast_dtype in columns_to_np_types.items():
+ # In case the collate_fn returns something strange
+ array = np.array(batch[col])
+ array = array.astype(cast_dtype)
+ out_batch[col] = array
+ else:
+ out_batch = []
+ for col, cast_dtype in columns_to_np_types.items():
+ # In case the collate_fn returns something strange
+ array = np.array(batch[col])
+ array = array.astype(cast_dtype)
+ out_batch.append(array)
+ return out_batch
+
+
+def dataset_to_tf(
+ dataset,
+ cols_to_retain,
+ collate_fn,
+ collate_fn_args,
+ columns_to_np_types,
+ output_signature,
+ shuffle,
+ batch_size,
+ drop_remainder,
+):
+ """Create a tf.data.Dataset from the underlying Dataset. This is a single-process method - the multiprocess
+ equivalent is multiprocess_dataset_to_tf.
+
+ Args:
+ dataset (`Dataset`): Dataset to wrap with tf.data.Dataset.
+ cols_to_retain (`List[str]`): Dataset column(s) to load in the
+ tf.data.Dataset. It is acceptable to include column names that are created by the `collate_fn` and
+ that do not exist in the original dataset.
+ collate_fn(`Callable`): A function or callable object (such as a `DataCollator`) that will collate
+ lists of samples into a batch.
+ collate_fn_args (`Dict`): A `dict` of keyword arguments to be passed to the
+ `collate_fn`. Can be empty.
+ columns_to_np_types (`Dict[str, np.dtype]`): A `dict` mapping column names to numpy dtypes.
+ output_signature (`Dict[str, tf.TensorSpec]`): A `dict` mapping column names to
+ `tf.TensorSpec` objects.
+ shuffle(`bool`): Shuffle the dataset order when loading. Recommended True for training, False for
+ validation/evaluation.
+ batch_size (`int`, default `None`): Size of batches to load from the dataset. Defaults to `None`, which implies that
+ the dataset won't be batched, but the returned dataset can be batched later with `tf_dataset.batch(batch_size)`.
+ drop_remainder(`bool`, default `None`): Drop the last incomplete batch when loading. If not provided,
+ defaults to the same setting as shuffle.
+
+ Returns:
+ `tf.data.Dataset`
+ """
+ if config.TF_AVAILABLE:
+ import tensorflow as tf
+ else:
+ raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.")
+
+ # TODO Matt: When our minimum Python version is 3.8 or higher, we can delete all of this and move everything
+ # to the NumPy multiprocessing path.
+ if hasattr(tf, "random_index_shuffle"):
+ random_index_shuffle = tf.random_index_shuffle
+ elif hasattr(tf.random.experimental, "index_shuffle"):
+ random_index_shuffle = tf.random.experimental.index_shuffle
+ else:
+ if len(dataset) > 10_000_000:
+ warnings.warn(
+ "to_tf_dataset() can be memory-inefficient on versions of TensorFlow older than 2.9. "
+ "If you are iterating over a dataset with a very large number of samples, consider "
+ "upgrading to TF >= 2.9."
+ )
+ random_index_shuffle = None
+
+ getter_fn = partial(
+ np_get_batch,
+ dataset=dataset,
+ cols_to_retain=cols_to_retain,
+ collate_fn=collate_fn,
+ collate_fn_args=collate_fn_args,
+ columns_to_np_types=columns_to_np_types,
+ return_dict=False,
+ )
+
+ # This works because dictionaries always output in the same order
+ tout = [tf.dtypes.as_dtype(dtype) for dtype in columns_to_np_types.values()]
+
+ @tf.function(input_signature=[tf.TensorSpec(None, tf.int64)])
+ def fetch_function(indices):
+ output = tf.py_function(
+ getter_fn,
+ inp=[indices],
+ Tout=tout,
+ )
+ return {key: output[i] for i, key in enumerate(columns_to_np_types.keys())}
+
+ tf_dataset = tf.data.Dataset.range(len(dataset))
+
+ if shuffle and random_index_shuffle is not None:
+ base_seed = tf.fill((3,), value=tf.cast(-1, dtype=tf.int64))
+
+ def scan_random_index(state, index):
+ if tf.reduce_all(state == -1):
+ # This generates a new random seed once per epoch only,
+ # to ensure that we iterate over each sample exactly once per epoch
+ state = tf.random.uniform(shape=(3,), maxval=2**62, dtype=tf.int64)
+ shuffled_index = random_index_shuffle(index=index, seed=state, max_index=len(dataset) - 1)
+ return state, shuffled_index
+
+ tf_dataset = tf_dataset.scan(base_seed, scan_random_index)
+ elif shuffle:
+ tf_dataset = tf_dataset.shuffle(tf_dataset.cardinality())
+
+ if batch_size is not None:
+ tf_dataset = tf_dataset.batch(batch_size, drop_remainder=drop_remainder)
+
+ tf_dataset = tf_dataset.map(fetch_function)
+
+ if batch_size is not None:
+
+ def ensure_shapes(input_dict):
+ return {key: tf.ensure_shape(val, output_signature[key].shape) for key, val in input_dict.items()}
+
+ else:
+ # Ensure shape but remove batch dimension of output_signature[key].shape
+ def ensure_shapes(input_dict):
+ return {key: tf.ensure_shape(val, output_signature[key].shape[1:]) for key, val in input_dict.items()}
+
+ return tf_dataset.map(ensure_shapes)
+
+
+class SharedMemoryContext:
+ # This is a context manager for creating shared memory that ensures cleanup happens even if a process is interrupted
+ # The process that creates shared memory is always the one responsible for unlinking it in the end
+ def __init__(self):
+ self.created_shms = []
+ self.opened_shms = []
+
+ def get_shm(self, name, size, create):
+ shm = SharedMemory(size=int(size), name=name, create=create)
+ if create:
+ # We only unlink the ones we created in this context
+ self.created_shms.append(shm)
+ else:
+ # If we didn't create it, we only close it when done, we don't unlink it
+ self.opened_shms.append(shm)
+ return shm
+
+ def get_array(self, name, shape, dtype, create):
+ shm = self.get_shm(name=name, size=np.prod(shape) * np.dtype(dtype).itemsize, create=create)
+ return np.ndarray(shape, dtype=dtype, buffer=shm.buf)
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ for shm in self.created_shms:
+ shm.close()
+ shm.unlink()
+ for shm in self.opened_shms:
+ shm.close()
+
+
+class NumpyMultiprocessingGenerator:
+ def __init__(
+ self,
+ dataset,
+ cols_to_retain,
+ collate_fn,
+ collate_fn_args,
+ columns_to_np_types,
+ output_signature,
+ shuffle,
+ batch_size,
+ drop_remainder,
+ num_workers,
+ ):
+ self.dataset = dataset
+ self.cols_to_retain = cols_to_retain
+ self.collate_fn = collate_fn
+ self.collate_fn_args = collate_fn_args
+ self.string_columns = [col for col, dtype in columns_to_np_types.items() if dtype is np.str_]
+ # Strings will be converted to arrays of single unicode chars, so that we can have a constant itemsize
+ self.columns_to_np_types = {
+ col: dtype if col not in self.string_columns else np.dtype("U1")
+ for col, dtype in columns_to_np_types.items()
+ }
+ self.output_signature = output_signature
+ self.shuffle = shuffle
+ self.batch_size = batch_size
+ self.drop_remainder = drop_remainder
+ self.num_workers = num_workers
+ # Because strings are converted to characters, we need to add one extra dimension to the shape
+ self.columns_to_ranks = {
+ col: int(spec.shape.rank) if col not in self.string_columns else int(spec.shape.rank) + 1
+ for col, spec in output_signature.items()
+ }
+
+ def __iter__(self):
+ # Make sure we only spawn workers if they have work to do
+ num_workers = min(self.num_workers, int(ceil(len(self.dataset) / self.batch_size)))
+ # Do the shuffling in iter so that it's done at the start of each epoch
+ per_worker_batches, final_batch, final_batch_worker = self.distribute_batches(
+ self.dataset, self.batch_size, self.drop_remainder, num_workers, self.shuffle
+ )
+ ctx = get_context("spawn")
+ names = []
+ shape_arrays = []
+ workers = []
+ array_ready_events = [ctx.Event() for _ in range(num_workers)]
+ array_loaded_events = [ctx.Event() for _ in range(num_workers)]
+
+ base_args = {
+ "dataset": self.dataset,
+ "cols_to_retain": self.cols_to_retain,
+ "collate_fn": self.collate_fn,
+ "collate_fn_args": self.collate_fn_args,
+ "columns_to_np_types": self.columns_to_np_types,
+ "columns_to_ranks": self.columns_to_ranks,
+ "string_columns": self.string_columns,
+ }
+ with SharedMemoryContext() as shm_ctx:
+ for i in range(num_workers):
+ worker_random_id = str(uuid4())
+ worker_name = f"dw_{i}_{worker_random_id}"[:10]
+ names.append(worker_name)
+
+ worker_shape_arrays = {
+ col: shm_ctx.get_array(f"{worker_name}_{col}_shape", shape=(rank,), dtype=np.int64, create=True)
+ for col, rank in self.columns_to_ranks.items()
+ }
+ shape_arrays.append(worker_shape_arrays)
+
+ worker_indices = per_worker_batches[i]
+ if i == final_batch_worker and final_batch is not None:
+ final_batch_arg = final_batch
+ else:
+ final_batch_arg = None
+ worker_kwargs = {
+ "worker_name": worker_name,
+ "indices": worker_indices,
+ "extra_batch": final_batch_arg,
+ "array_ready_event": array_ready_events[i],
+ "array_loaded_event": array_loaded_events[i],
+ **base_args,
+ }
+ worker = ctx.Process(target=self.worker_loop, kwargs=worker_kwargs, daemon=True)
+ worker.start()
+ workers.append(worker)
+
+ end_signal_received = False
+ while not end_signal_received:
+ for i in range(num_workers):
+ if not array_ready_events[i].wait(timeout=60):
+ raise TimeoutError("Data loading worker timed out!")
+ array_ready_events[i].clear()
+ array_shapes = shape_arrays[i]
+ if any(np.any(shape < 0) for shape in array_shapes.values()):
+ # Child processes send negative array shapes to indicate
+ # that no more data is going to be sent
+ end_signal_received = True
+ break
+ # Matt: Because array shapes are variable we recreate the shared memory each iteration.
+ # I suspect repeatedly opening lots of shared memory is the bottleneck for the parent process.
+ # A future optimization, at the cost of some code complexity, could be to reuse shared memory
+ # between iterations, but this would require knowing in advance the maximum size, or having
+ # a system to only create a new memory block when a new maximum size is seen.
+ # Another potential optimization would be to figure out which memory copies are necessary,
+ # or whether we can yield objects straight out of shared memory.
+ with SharedMemoryContext() as batch_shm_ctx:
+ # This memory context only lasts long enough to copy everything out of the batch
+ arrays = {
+ col: batch_shm_ctx.get_array(
+ f"{names[i]}_{col}",
+ shape=shape,
+ dtype=self.columns_to_np_types[col],
+ create=False,
+ )
+ for col, shape in array_shapes.items()
+ }
+ # Copy everything out of shm because the memory
+ # will be unlinked by the child process at some point
+ arrays = {col: np.copy(arr) for col, arr in arrays.items()}
+ # Now we convert any unicode char arrays to strings
+ for string_col in self.string_columns:
+ arrays[string_col] = (
+ arrays[string_col].view(f"U{arrays[string_col].shape[-1]}").squeeze(-1)
+ )
+ yield arrays
+ array_loaded_events[i].set()
+ # Now we just do some cleanup
+ # Shared memory is cleaned up by the context manager, so we just make sure workers finish
+ for worker in workers:
+ worker.join()
+
+ def __call__(self):
+ return self
+
+ @staticmethod
+ def worker_loop(
+ dataset,
+ cols_to_retain,
+ collate_fn,
+ collate_fn_args,
+ columns_to_np_types,
+ columns_to_ranks,
+ string_columns,
+ indices,
+ extra_batch,
+ worker_name,
+ array_ready_event,
+ array_loaded_event,
+ ):
+ os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
+
+ if config.TF_AVAILABLE:
+ import tensorflow as tf
+ else:
+ raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.")
+
+ tf.config.set_visible_devices([], "GPU") # Make sure workers don't try to allocate GPU memory
+
+ def send_batch_to_parent(indices):
+ batch = np_get_batch(
+ indices=indices,
+ dataset=dataset,
+ cols_to_retain=cols_to_retain,
+ collate_fn=collate_fn,
+ collate_fn_args=collate_fn_args,
+ columns_to_np_types=columns_to_np_types,
+ return_dict=True,
+ )
+
+ # Now begins the fun part where we start shovelling shared memory at the parent process
+ out_arrays = {}
+ with SharedMemoryContext() as batch_shm_ctx:
+ # The batch shared memory context exists only as long as it takes for the parent process
+ # to read everything, after which it cleans everything up again
+ for col, cast_dtype in columns_to_np_types.items():
+ # Everything has to be np.array for this to work, even if the collate_fn is giving us tf.Tensor
+ array = batch[col]
+ if col in string_columns:
+ # We can't send unicode arrays over shared memory, so we convert to single chars ("U1")
+ # which have a fixed width of 4 bytes. The parent process will convert these back to strings.
+ array = array.view("U1").reshape(array.shape + (-1,))
+ shape_arrays[col][:] = array.shape
+ out_arrays[col] = batch_shm_ctx.get_array(
+ f"{worker_name}_{col}", shape=array.shape, dtype=cast_dtype, create=True
+ )
+ out_arrays[col][:] = array
+
+ array_ready_event.set()
+ array_loaded_event.wait()
+ array_loaded_event.clear()
+
+ with SharedMemoryContext() as shm_ctx:
+ shape_arrays = {
+ col: shm_ctx.get_array(f"{worker_name}_{col}_shape", shape=(rank,), dtype=np.int64, create=False)
+ for col, rank in columns_to_ranks.items()
+ }
+
+ for batch in indices:
+ send_batch_to_parent(batch)
+ if extra_batch is not None:
+ send_batch_to_parent(extra_batch)
+ # Now we send a batsignal to the parent process that we're done
+ for col, array in shape_arrays.items():
+ array[:] = -1
+ array_ready_event.set()
+
+ @staticmethod
+ def distribute_batches(dataset, batch_size, drop_remainder, num_workers, shuffle):
+ indices = np.arange(len(dataset))
+ if shuffle:
+ np.random.shuffle(indices)
+ num_samples = len(indices)
+ # We distribute the batches so that reading from the workers in round-robin order yields the exact
+ # order specified in indices. This is only important when shuffle is False, but we do it regardless.
+ incomplete_batch_cutoff = num_samples - (num_samples % batch_size)
+ indices, last_incomplete_batch = np.split(indices, [incomplete_batch_cutoff])
+ if drop_remainder or len(last_incomplete_batch) == 0:
+ last_incomplete_batch = None
+
+ indices = indices.reshape(-1, batch_size)
+ num_batches = len(indices)
+ final_batches_cutoff = num_batches - (num_batches % num_workers)
+ indices, final_batches = np.split(indices, [final_batches_cutoff])
+ indices = indices.reshape(-1, num_workers, batch_size)
+
+ per_worker_indices = np.split(indices, indices.shape[1], axis=1)
+ per_worker_indices = [np.squeeze(worker_indices, 1) for worker_indices in per_worker_indices]
+ # Distribute the final batches to the first workers
+ for i in range(len(final_batches)):
+ # len(final_batches) can be zero, and is always less than num_workers
+ per_worker_indices[i] = np.concatenate([per_worker_indices[i], final_batches[i].reshape(1, -1)], axis=0)
+ # Add the last incomplete batch to the next worker, which might be the first worker
+ if last_incomplete_batch is not None:
+ incomplete_batch_worker_idx = len(final_batches)
+ else:
+ incomplete_batch_worker_idx = None
+ return per_worker_indices, last_incomplete_batch, incomplete_batch_worker_idx
+
+
+def multiprocess_dataset_to_tf(
+ dataset,
+ cols_to_retain,
+ collate_fn,
+ collate_fn_args,
+ columns_to_np_types,
+ output_signature,
+ shuffle,
+ batch_size,
+ drop_remainder,
+ num_workers,
+):
+ """Create a tf.data.Dataset from the underlying Dataset. This is a multi-process method - the single-process
+ equivalent is dataset_to_tf.
+
+ Args:
+ dataset (`Dataset`): Dataset to wrap with tf.data.Dataset.
+ cols_to_retain (`List[str]`): Dataset column(s) to load in the
+ tf.data.Dataset. It is acceptable to include column names that are created by the `collate_fn` and
+ that do not exist in the original dataset.
+ collate_fn(`Callable`): A function or callable object (such as a `DataCollator`) that will collate
+ lists of samples into a batch.
+ collate_fn_args (`Dict`): A `dict` of keyword arguments to be passed to the
+ `collate_fn`. Can be empty.
+ columns_to_np_types (`Dict[str, np.dtype]`): A `dict` mapping column names to numpy dtypes.
+ output_signature (`Dict[str, tf.TensorSpec]`): A `dict` mapping column names to
+ `tf.TensorSpec` objects.
+ shuffle(`bool`): Shuffle the dataset order when loading. Recommended True for training, False for
+ validation/evaluation.
+ batch_size (`int`, default `None`): Size of batches to load from the dataset. Defaults to `None`, which implies that
+ the dataset won't be batched, but the returned dataset can be batched later with `tf_dataset.batch(batch_size)`.
+ drop_remainder(`bool`, default `None`): Drop the last incomplete batch when loading. If not provided,
+ defaults to the same setting as shuffle.
+ num_workers (`int`): Number of workers to use for loading the dataset. Should be >= 1.
+
+ Returns:
+ `tf.data.Dataset`
+ """
+ if config.TF_AVAILABLE:
+ import tensorflow as tf
+ else:
+ raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.")
+
+ data_generator = NumpyMultiprocessingGenerator(
+ dataset=dataset,
+ cols_to_retain=cols_to_retain,
+ collate_fn=collate_fn,
+ collate_fn_args=collate_fn_args,
+ columns_to_np_types=columns_to_np_types,
+ output_signature=output_signature,
+ shuffle=shuffle,
+ batch_size=batch_size,
+ drop_remainder=drop_remainder,
+ num_workers=num_workers,
+ )
+
+ tf_dataset = tf.data.Dataset.from_generator(data_generator, output_signature=output_signature)
+ if drop_remainder:
+ dataset_length = int(len(dataset) // batch_size)
+ else:
+ dataset_length = int(ceil(len(dataset) / batch_size))
+ return tf_dataset.apply(tf.data.experimental.assert_cardinality(dataset_length))
diff --git a/venv/Lib/site-packages/datasets/utils/tqdm.py b/venv/Lib/site-packages/datasets/utils/tqdm.py
new file mode 100644
index 0000000000000000000000000000000000000000..88ab7c067abf94290e16c206ef50bdcaa7f70d3b
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/tqdm.py
@@ -0,0 +1,135 @@
+"""Utility helpers to handle progress bars in `datasets`.
+
+Example:
+ 1. Use `datasets.utils.tqdm` as you would use `tqdm.tqdm` or `tqdm.auto.tqdm`.
+ 2. To disable progress bars, either use `disable_progress_bars()` helper or set the
+ environment variable `HF_DATASETS_DISABLE_PROGRESS_BARS` to 1.
+ 3. To re-enable progress bars, use `enable_progress_bars()`.
+ 4. To check whether progress bars are disabled, use `are_progress_bars_disabled()`.
+
+NOTE: Environment variable `HF_DATASETS_DISABLE_PROGRESS_BARS` has the priority.
+
+Example:
+ ```py
+ from datasets.utils import (
+ are_progress_bars_disabled,
+ disable_progress_bars,
+ enable_progress_bars,
+ tqdm,
+ )
+
+ # Disable progress bars globally
+ disable_progress_bars()
+
+ # Use as normal `tqdm`
+ for _ in tqdm(range(5)):
+ do_something()
+
+ # Still not showing progress bars, as `disable=False` is overwritten to `True`.
+ for _ in tqdm(range(5), disable=False):
+ do_something()
+
+ are_progress_bars_disabled() # True
+
+ # Re-enable progress bars globally
+ enable_progress_bars()
+
+ # Progress bar will be shown !
+ for _ in tqdm(range(5)):
+ do_something()
+ ```
+"""
+
+import os
+import warnings
+
+from tqdm.auto import tqdm as old_tqdm
+
+from ..config import HF_DATASETS_DISABLE_PROGRESS_BARS
+
+
+# `HF_DATASETS_DISABLE_PROGRESS_BARS` is `Optional[bool]` while `_hf_datasets_progress_bars_disabled`
+# is a `bool`. If `HF_DATASETS_DISABLE_PROGRESS_BARS` is set to True or False, it has priority.
+# If `HF_DATASETS_DISABLE_PROGRESS_BARS` is None, it means the user have not set the
+# environment variable and is free to enable/disable progress bars programmatically.
+# TL;DR: env variable has priority over code.
+#
+# By default, progress bars are enabled.
+_hf_datasets_progress_bars_disabled: bool = HF_DATASETS_DISABLE_PROGRESS_BARS or False
+
+
+def disable_progress_bars() -> None:
+ """
+ Disable globally progress bars used in `datasets` except if `HF_DATASETS_DISABLE_PROGRESS_BARS` environment
+ variable has been set.
+
+ Use [`~utils.enable_progress_bars`] to re-enable them.
+ """
+ if HF_DATASETS_DISABLE_PROGRESS_BARS is False:
+ warnings.warn(
+ "Cannot disable progress bars: environment variable `HF_DATASETS_DISABLE_PROGRESS_BARS=0` is set and has"
+ " priority."
+ )
+ return
+ global _hf_datasets_progress_bars_disabled
+ _hf_datasets_progress_bars_disabled = True
+
+
+def enable_progress_bars() -> None:
+ """
+ Enable globally progress bars used in `datasets` except if `HF_DATASETS_DISABLE_PROGRESS_BARS` environment
+ variable has been set.
+
+ Use [`~utils.disable_progress_bars`] to disable them.
+ """
+ if HF_DATASETS_DISABLE_PROGRESS_BARS is True:
+ warnings.warn(
+ "Cannot enable progress bars: environment variable `HF_DATASETS_DISABLE_PROGRESS_BARS=1` is set and has"
+ " priority."
+ )
+ return
+ global _hf_datasets_progress_bars_disabled
+ _hf_datasets_progress_bars_disabled = False
+
+
+def are_progress_bars_disabled() -> bool:
+ """Return whether progress bars are globally disabled or not.
+
+ Progress bars used in `datasets` can be enable or disabled globally using [`~utils.enable_progress_bars`]
+ and [`~utils.disable_progress_bars`] or by setting `HF_DATASETS_DISABLE_PROGRESS_BARS` as environment variable.
+ """
+ global _hf_datasets_progress_bars_disabled
+ return _hf_datasets_progress_bars_disabled
+
+
+class tqdm(old_tqdm):
+ """
+ Class to override `disable` argument in case progress bars are globally disabled.
+
+ Taken from https://github.com/tqdm/tqdm/issues/619#issuecomment-619639324.
+ """
+
+ def __init__(self, *args, **kwargs):
+ if are_progress_bars_disabled():
+ kwargs["disable"] = True
+ elif kwargs.get("disable") is None and os.getenv("TQDM_POSITION") == "-1":
+ # Force-enable progress bars in cloud environments when disable=None
+ kwargs["disable"] = False
+ super().__init__(*args, **kwargs)
+
+ def __delattr__(self, attr: str) -> None:
+ """Fix for https://github.com/huggingface/datasets/issues/6066"""
+ try:
+ super().__delattr__(attr)
+ except AttributeError:
+ if attr != "_lock":
+ raise
+
+
+# backward compatibility
+enable_progress_bar = enable_progress_bars
+disable_progress_bar = disable_progress_bars
+
+
+def is_progress_bar_enabled():
+ return not are_progress_bars_disabled()
diff --git a/venv/Lib/site-packages/datasets/utils/track.py b/venv/Lib/site-packages/datasets/utils/track.py
new file mode 100644
index 0000000000000000000000000000000000000000..ffd3434af790de63fdd046ad8f93c3067c6e6da8
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/track.py
@@ -0,0 +1,61 @@
+from collections.abc import Iterable, Iterator
+
+
+class tracked_str(str):
+ origins = {}
+
+ def set_origin(self, origin: str):
+ if super().__repr__() not in self.origins:
+ self.origins[super().__repr__()] = origin
+
+ def get_origin(self):
+ return self.origins.get(super().__repr__(), str(self))
+
+ def __repr__(self) -> str:
+ if super().__repr__() not in self.origins or self.origins[super().__repr__()] == self:
+ return super().__repr__()
+ else:
+ return f"{str(self)} (origin={self.origins[super().__repr__()]})"
+
+
+class tracked_list(list):
+ def __init__(self, *args, **kwargs) -> None:
+ super().__init__(*args, **kwargs)
+ self.last_item = None
+
+ def __iter__(self) -> Iterator:
+ for x in super().__iter__():
+ self.last_item = x
+ yield x
+ self.last_item = None
+
+ def __repr__(self) -> str:
+ if self.last_item is None:
+ return super().__repr__()
+ else:
+ return f"{self.__class__.__name__}(current={self.last_item})"
+
+
+class TrackedIterableFromGenerator(Iterable):
+ """Utility class to create an iterable from a generator function, in order to reset the generator when needed."""
+
+ def __init__(self, generator, *args):
+ super().__init__()
+ self.generator = generator
+ self.args = args
+ self.last_item = None
+
+ def __iter__(self):
+ for x in self.generator(*self.args):
+ self.last_item = x
+ yield x
+ self.last_item = None
+
+ def __repr__(self) -> str:
+ if self.last_item is None:
+ return super().__repr__()
+ else:
+ return f"{self.__class__.__name__}(current={self.last_item})"
+
+ def __reduce__(self):
+ return (self.__class__, (self.generator, *self.args))
diff --git a/venv/Lib/site-packages/datasets/utils/typing.py b/venv/Lib/site-packages/datasets/utils/typing.py
new file mode 100644
index 0000000000000000000000000000000000000000..4736d74cbb575bd0e12f625c520202a8c74b3ddc
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/typing.py
@@ -0,0 +1,9 @@
+import os
+from typing import TypeVar, Union
+
+
+T = TypeVar("T")
+
+ListLike = Union[list[T], tuple[T, ...]]
+NestedDataStructureLike = Union[T, list[T], dict[str, T]]
+PathLike = Union[str, bytes, os.PathLike]
diff --git a/venv/Lib/site-packages/datasets/utils/version.py b/venv/Lib/site-packages/datasets/utils/version.py
new file mode 100644
index 0000000000000000000000000000000000000000..75cf4c39d5f9b916ade8f8d88a0f0ca9e5769217
--- /dev/null
+++ b/venv/Lib/site-packages/datasets/utils/version.py
@@ -0,0 +1,106 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Version utils."""
+
+import dataclasses
+import re
+from dataclasses import dataclass
+from functools import total_ordering
+from typing import Optional, Union
+
+
+_VERSION_REG = re.compile(r"^(?P\d+)" r"\.(?P\d+)" r"\.(?P\d+)$")
+
+
+@total_ordering
+@dataclass
+class Version:
+ """Dataset version `MAJOR.MINOR.PATCH`.
+
+ Args:
+ version_str (`str`):
+ The dataset version.
+ description (`str`):
+ A description of what is new in this version.
+ major (`str`):
+ minor (`str`):
+ patch (`str`):
+
+ Example:
+
+ ```py
+ >>> VERSION = datasets.Version("1.0.0")
+ ```
+ """
+
+ version_str: str
+ description: Optional[str] = None
+ major: Optional[Union[str, int]] = None
+ minor: Optional[Union[str, int]] = None
+ patch: Optional[Union[str, int]] = None
+
+ def __post_init__(self):
+ self.major, self.minor, self.patch = _str_to_version_tuple(self.version_str)
+
+ def __repr__(self):
+ return f"{self.tuple[0]}.{self.tuple[1]}.{self.tuple[2]}"
+
+ @property
+ def tuple(self):
+ return self.major, self.minor, self.patch
+
+ def _validate_operand(self, other):
+ if isinstance(other, str):
+ return Version(other)
+ elif isinstance(other, Version):
+ return other
+ raise TypeError(f"{other} (type {type(other)}) cannot be compared to version.")
+
+ def __eq__(self, other):
+ try:
+ other = self._validate_operand(other)
+ except (TypeError, ValueError):
+ return False
+ else:
+ return self.tuple == other.tuple
+
+ def __lt__(self, other):
+ other = self._validate_operand(other)
+ return self.tuple < other.tuple
+
+ def __hash__(self):
+ return hash(_version_tuple_to_str(self.tuple))
+
+ @classmethod
+ def from_dict(cls, dic):
+ field_names = {f.name for f in dataclasses.fields(cls)}
+ return cls(**{k: v for k, v in dic.items() if k in field_names})
+
+ def _to_yaml_string(self) -> str:
+ return self.version_str
+
+
+def _str_to_version_tuple(version_str):
+ """Return the tuple (major, minor, patch) version extracted from the str."""
+ res = _VERSION_REG.match(version_str)
+ if not res:
+ raise ValueError(f"Invalid version '{version_str}'. Format should be x.y.z with {{x,y,z}} being digits.")
+ return tuple(int(v) for v in [res.group("major"), res.group("minor"), res.group("patch")])
+
+
+def _version_tuple_to_str(version_tuple):
+ """Return the str version from the version tuple (major, minor, patch)."""
+ return ".".join(str(v) for v in version_tuple)
diff --git a/venv/Lib/site-packages/dateutil/__init__.py b/venv/Lib/site-packages/dateutil/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..a2c19c06fe14476a9bfa4f1f60de7a997a41191c
--- /dev/null
+++ b/venv/Lib/site-packages/dateutil/__init__.py
@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+import sys
+
+try:
+ from ._version import version as __version__
+except ImportError:
+ __version__ = 'unknown'
+
+__all__ = ['easter', 'parser', 'relativedelta', 'rrule', 'tz',
+ 'utils', 'zoneinfo']
+
+def __getattr__(name):
+ import importlib
+
+ if name in __all__:
+ return importlib.import_module("." + name, __name__)
+ raise AttributeError(
+ "module {!r} has not attribute {!r}".format(__name__, name)
+ )
+
+
+def __dir__():
+ # __dir__ should include all the lazy-importable modules as well.
+ return [x for x in globals() if x not in sys.modules] + __all__
diff --git a/venv/Lib/site-packages/dateutil/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/dateutil/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..a90c8a0498fc0dd0972583102dbc5beb044f6f9e
Binary files /dev/null and b/venv/Lib/site-packages/dateutil/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dateutil/__pycache__/_common.cpython-311.pyc b/venv/Lib/site-packages/dateutil/__pycache__/_common.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4308886ff45bae94edd29226aace85bb5a40b1d3
Binary files /dev/null and b/venv/Lib/site-packages/dateutil/__pycache__/_common.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dateutil/__pycache__/_version.cpython-311.pyc b/venv/Lib/site-packages/dateutil/__pycache__/_version.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c67ee724fe2d8195168fb55fe99ae981a18381d4
Binary files /dev/null and b/venv/Lib/site-packages/dateutil/__pycache__/_version.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dateutil/__pycache__/easter.cpython-311.pyc b/venv/Lib/site-packages/dateutil/__pycache__/easter.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..dd5a5b53a0958515cedd7564e779122bb984f707
Binary files /dev/null and b/venv/Lib/site-packages/dateutil/__pycache__/easter.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dateutil/__pycache__/relativedelta.cpython-311.pyc b/venv/Lib/site-packages/dateutil/__pycache__/relativedelta.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e72cbeea1902e9d38a00bbb96c88351de8f6638e
Binary files /dev/null and b/venv/Lib/site-packages/dateutil/__pycache__/relativedelta.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dateutil/__pycache__/rrule.cpython-311.pyc b/venv/Lib/site-packages/dateutil/__pycache__/rrule.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c3ccb908e014ce76579a78a3c3b3ae20b73e5e55
Binary files /dev/null and b/venv/Lib/site-packages/dateutil/__pycache__/rrule.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dateutil/__pycache__/tzwin.cpython-311.pyc b/venv/Lib/site-packages/dateutil/__pycache__/tzwin.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e6d6affc999f580f14000c51c465e0584dfe9b75
Binary files /dev/null and b/venv/Lib/site-packages/dateutil/__pycache__/tzwin.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dateutil/__pycache__/utils.cpython-311.pyc b/venv/Lib/site-packages/dateutil/__pycache__/utils.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..db5e9aaa09a38944e24f1b9a1c713fc06b8dedd2
Binary files /dev/null and b/venv/Lib/site-packages/dateutil/__pycache__/utils.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dateutil/_common.py b/venv/Lib/site-packages/dateutil/_common.py
new file mode 100644
index 0000000000000000000000000000000000000000..4eb2659bd2986125fcfb4afea5bae9efc2dcd1a0
--- /dev/null
+++ b/venv/Lib/site-packages/dateutil/_common.py
@@ -0,0 +1,43 @@
+"""
+Common code used in multiple modules.
+"""
+
+
+class weekday(object):
+ __slots__ = ["weekday", "n"]
+
+ def __init__(self, weekday, n=None):
+ self.weekday = weekday
+ self.n = n
+
+ def __call__(self, n):
+ if n == self.n:
+ return self
+ else:
+ return self.__class__(self.weekday, n)
+
+ def __eq__(self, other):
+ try:
+ if self.weekday != other.weekday or self.n != other.n:
+ return False
+ except AttributeError:
+ return False
+ return True
+
+ def __hash__(self):
+ return hash((
+ self.weekday,
+ self.n,
+ ))
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ def __repr__(self):
+ s = ("MO", "TU", "WE", "TH", "FR", "SA", "SU")[self.weekday]
+ if not self.n:
+ return s
+ else:
+ return "%s(%+d)" % (s, self.n)
+
+# vim:ts=4:sw=4:et
diff --git a/venv/Lib/site-packages/dateutil/_version.py b/venv/Lib/site-packages/dateutil/_version.py
new file mode 100644
index 0000000000000000000000000000000000000000..ddda98098527a73348e694c2edb691fd625475fc
--- /dev/null
+++ b/venv/Lib/site-packages/dateutil/_version.py
@@ -0,0 +1,4 @@
+# file generated by setuptools_scm
+# don't change, don't track in version control
+__version__ = version = '2.9.0.post0'
+__version_tuple__ = version_tuple = (2, 9, 0)
diff --git a/venv/Lib/site-packages/dateutil/easter.py b/venv/Lib/site-packages/dateutil/easter.py
new file mode 100644
index 0000000000000000000000000000000000000000..f74d1f7442473997245ac683b8a269a3574d1ba4
--- /dev/null
+++ b/venv/Lib/site-packages/dateutil/easter.py
@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+"""
+This module offers a generic Easter computing method for any given year, using
+Western, Orthodox or Julian algorithms.
+"""
+
+import datetime
+
+__all__ = ["easter", "EASTER_JULIAN", "EASTER_ORTHODOX", "EASTER_WESTERN"]
+
+EASTER_JULIAN = 1
+EASTER_ORTHODOX = 2
+EASTER_WESTERN = 3
+
+
+def easter(year, method=EASTER_WESTERN):
+ """
+ This method was ported from the work done by GM Arts,
+ on top of the algorithm by Claus Tondering, which was
+ based in part on the algorithm of Ouding (1940), as
+ quoted in "Explanatory Supplement to the Astronomical
+ Almanac", P. Kenneth Seidelmann, editor.
+
+ This algorithm implements three different Easter
+ calculation methods:
+
+ 1. Original calculation in Julian calendar, valid in
+ dates after 326 AD
+ 2. Original method, with date converted to Gregorian
+ calendar, valid in years 1583 to 4099
+ 3. Revised method, in Gregorian calendar, valid in
+ years 1583 to 4099 as well
+
+ These methods are represented by the constants:
+
+ * ``EASTER_JULIAN = 1``
+ * ``EASTER_ORTHODOX = 2``
+ * ``EASTER_WESTERN = 3``
+
+ The default method is method 3.
+
+ More about the algorithm may be found at:
+
+ `GM Arts: Easter Algorithms `_
+
+ and
+
+ `The Calendar FAQ: Easter `_
+
+ """
+
+ if not (1 <= method <= 3):
+ raise ValueError("invalid method")
+
+ # g - Golden year - 1
+ # c - Century
+ # h - (23 - Epact) mod 30
+ # i - Number of days from March 21 to Paschal Full Moon
+ # j - Weekday for PFM (0=Sunday, etc)
+ # p - Number of days from March 21 to Sunday on or before PFM
+ # (-6 to 28 methods 1 & 3, to 56 for method 2)
+ # e - Extra days to add for method 2 (converting Julian
+ # date to Gregorian date)
+
+ y = year
+ g = y % 19
+ e = 0
+ if method < 3:
+ # Old method
+ i = (19*g + 15) % 30
+ j = (y + y//4 + i) % 7
+ if method == 2:
+ # Extra dates to convert Julian to Gregorian date
+ e = 10
+ if y > 1600:
+ e = e + y//100 - 16 - (y//100 - 16)//4
+ else:
+ # New method
+ c = y//100
+ h = (c - c//4 - (8*c + 13)//25 + 19*g + 15) % 30
+ i = h - (h//28)*(1 - (h//28)*(29//(h + 1))*((21 - g)//11))
+ j = (y + y//4 + i + 2 - c + c//4) % 7
+
+ # p can be from -6 to 56 corresponding to dates 22 March to 23 May
+ # (later dates apply to method 2, although 23 May never actually occurs)
+ p = i - j + e
+ d = 1 + (p + 27 + (p + 6)//40) % 31
+ m = 3 + (p + 26)//30
+ return datetime.date(int(y), int(m), int(d))
diff --git a/venv/Lib/site-packages/dateutil/parser/__init__.py b/venv/Lib/site-packages/dateutil/parser/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..d174b0e4dcc472999b75e55ebb88af320ae38081
--- /dev/null
+++ b/venv/Lib/site-packages/dateutil/parser/__init__.py
@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+from ._parser import parse, parser, parserinfo, ParserError
+from ._parser import DEFAULTPARSER, DEFAULTTZPARSER
+from ._parser import UnknownTimezoneWarning
+
+from ._parser import __doc__
+
+from .isoparser import isoparser, isoparse
+
+__all__ = ['parse', 'parser', 'parserinfo',
+ 'isoparse', 'isoparser',
+ 'ParserError',
+ 'UnknownTimezoneWarning']
+
+
+###
+# Deprecate portions of the private interface so that downstream code that
+# is improperly relying on it is given *some* notice.
+
+
+def __deprecated_private_func(f):
+ from functools import wraps
+ import warnings
+
+ msg = ('{name} is a private function and may break without warning, '
+ 'it will be moved and or renamed in future versions.')
+ msg = msg.format(name=f.__name__)
+
+ @wraps(f)
+ def deprecated_func(*args, **kwargs):
+ warnings.warn(msg, DeprecationWarning)
+ return f(*args, **kwargs)
+
+ return deprecated_func
+
+def __deprecate_private_class(c):
+ import warnings
+
+ msg = ('{name} is a private class and may break without warning, '
+ 'it will be moved and or renamed in future versions.')
+ msg = msg.format(name=c.__name__)
+
+ class private_class(c):
+ __doc__ = c.__doc__
+
+ def __init__(self, *args, **kwargs):
+ warnings.warn(msg, DeprecationWarning)
+ super(private_class, self).__init__(*args, **kwargs)
+
+ private_class.__name__ = c.__name__
+
+ return private_class
+
+
+from ._parser import _timelex, _resultbase
+from ._parser import _tzparser, _parsetz
+
+_timelex = __deprecate_private_class(_timelex)
+_tzparser = __deprecate_private_class(_tzparser)
+_resultbase = __deprecate_private_class(_resultbase)
+_parsetz = __deprecated_private_func(_parsetz)
diff --git a/venv/Lib/site-packages/dateutil/parser/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/dateutil/parser/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..67758237ccc6d7cd3435e96ea5711a59fabfc0b1
Binary files /dev/null and b/venv/Lib/site-packages/dateutil/parser/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dateutil/parser/__pycache__/_parser.cpython-311.pyc b/venv/Lib/site-packages/dateutil/parser/__pycache__/_parser.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7f98b6a4dd2fdaaa2304ac7dd923c1a22f5049b8
Binary files /dev/null and b/venv/Lib/site-packages/dateutil/parser/__pycache__/_parser.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dateutil/parser/__pycache__/isoparser.cpython-311.pyc b/venv/Lib/site-packages/dateutil/parser/__pycache__/isoparser.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6cc2bd6b06d6a76c2331f42ddca405c363a5cfc0
Binary files /dev/null and b/venv/Lib/site-packages/dateutil/parser/__pycache__/isoparser.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dateutil/parser/_parser.py b/venv/Lib/site-packages/dateutil/parser/_parser.py
new file mode 100644
index 0000000000000000000000000000000000000000..37d1663b2f72447800d9a553929e3de932244289
--- /dev/null
+++ b/venv/Lib/site-packages/dateutil/parser/_parser.py
@@ -0,0 +1,1613 @@
+# -*- coding: utf-8 -*-
+"""
+This module offers a generic date/time string parser which is able to parse
+most known formats to represent a date and/or time.
+
+This module attempts to be forgiving with regards to unlikely input formats,
+returning a datetime object even for dates which are ambiguous. If an element
+of a date/time stamp is omitted, the following rules are applied:
+
+- If AM or PM is left unspecified, a 24-hour clock is assumed, however, an hour
+ on a 12-hour clock (``0 <= hour <= 12``) *must* be specified if AM or PM is
+ specified.
+- If a time zone is omitted, a timezone-naive datetime is returned.
+
+If any other elements are missing, they are taken from the
+:class:`datetime.datetime` object passed to the parameter ``default``. If this
+results in a day number exceeding the valid number of days per month, the
+value falls back to the end of the month.
+
+Additional resources about date/time string formats can be found below:
+
+- `A summary of the international standard date and time notation
+ `_
+- `W3C Date and Time Formats `_
+- `Time Formats (Planetary Rings Node) `_
+- `CPAN ParseDate module
+ `_
+- `Java SimpleDateFormat Class
+ `_
+"""
+from __future__ import unicode_literals
+
+import datetime
+import re
+import string
+import time
+import warnings
+
+from calendar import monthrange
+from io import StringIO
+
+import six
+from six import integer_types, text_type
+
+from decimal import Decimal
+
+from warnings import warn
+
+from .. import relativedelta
+from .. import tz
+
+__all__ = ["parse", "parserinfo", "ParserError"]
+
+
+# TODO: pandas.core.tools.datetimes imports this explicitly. Might be worth
+# making public and/or figuring out if there is something we can
+# take off their plate.
+class _timelex(object):
+ # Fractional seconds are sometimes split by a comma
+ _split_decimal = re.compile("([.,])")
+
+ def __init__(self, instream):
+ if isinstance(instream, (bytes, bytearray)):
+ instream = instream.decode()
+
+ if isinstance(instream, text_type):
+ instream = StringIO(instream)
+ elif getattr(instream, 'read', None) is None:
+ raise TypeError('Parser must be a string or character stream, not '
+ '{itype}'.format(itype=instream.__class__.__name__))
+
+ self.instream = instream
+ self.charstack = []
+ self.tokenstack = []
+ self.eof = False
+
+ def get_token(self):
+ """
+ This function breaks the time string into lexical units (tokens), which
+ can be parsed by the parser. Lexical units are demarcated by changes in
+ the character set, so any continuous string of letters is considered
+ one unit, any continuous string of numbers is considered one unit.
+
+ The main complication arises from the fact that dots ('.') can be used
+ both as separators (e.g. "Sep.20.2009") or decimal points (e.g.
+ "4:30:21.447"). As such, it is necessary to read the full context of
+ any dot-separated strings before breaking it into tokens; as such, this
+ function maintains a "token stack", for when the ambiguous context
+ demands that multiple tokens be parsed at once.
+ """
+ if self.tokenstack:
+ return self.tokenstack.pop(0)
+
+ seenletters = False
+ token = None
+ state = None
+
+ while not self.eof:
+ # We only realize that we've reached the end of a token when we
+ # find a character that's not part of the current token - since
+ # that character may be part of the next token, it's stored in the
+ # charstack.
+ if self.charstack:
+ nextchar = self.charstack.pop(0)
+ else:
+ nextchar = self.instream.read(1)
+ while nextchar == '\x00':
+ nextchar = self.instream.read(1)
+
+ if not nextchar:
+ self.eof = True
+ break
+ elif not state:
+ # First character of the token - determines if we're starting
+ # to parse a word, a number or something else.
+ token = nextchar
+ if self.isword(nextchar):
+ state = 'a'
+ elif self.isnum(nextchar):
+ state = '0'
+ elif self.isspace(nextchar):
+ token = ' '
+ break # emit token
+ else:
+ break # emit token
+ elif state == 'a':
+ # If we've already started reading a word, we keep reading
+ # letters until we find something that's not part of a word.
+ seenletters = True
+ if self.isword(nextchar):
+ token += nextchar
+ elif nextchar == '.':
+ token += nextchar
+ state = 'a.'
+ else:
+ self.charstack.append(nextchar)
+ break # emit token
+ elif state == '0':
+ # If we've already started reading a number, we keep reading
+ # numbers until we find something that doesn't fit.
+ if self.isnum(nextchar):
+ token += nextchar
+ elif nextchar == '.' or (nextchar == ',' and len(token) >= 2):
+ token += nextchar
+ state = '0.'
+ else:
+ self.charstack.append(nextchar)
+ break # emit token
+ elif state == 'a.':
+ # If we've seen some letters and a dot separator, continue
+ # parsing, and the tokens will be broken up later.
+ seenletters = True
+ if nextchar == '.' or self.isword(nextchar):
+ token += nextchar
+ elif self.isnum(nextchar) and token[-1] == '.':
+ token += nextchar
+ state = '0.'
+ else:
+ self.charstack.append(nextchar)
+ break # emit token
+ elif state == '0.':
+ # If we've seen at least one dot separator, keep going, we'll
+ # break up the tokens later.
+ if nextchar == '.' or self.isnum(nextchar):
+ token += nextchar
+ elif self.isword(nextchar) and token[-1] == '.':
+ token += nextchar
+ state = 'a.'
+ else:
+ self.charstack.append(nextchar)
+ break # emit token
+
+ if (state in ('a.', '0.') and (seenletters or token.count('.') > 1 or
+ token[-1] in '.,')):
+ l = self._split_decimal.split(token)
+ token = l[0]
+ for tok in l[1:]:
+ if tok:
+ self.tokenstack.append(tok)
+
+ if state == '0.' and token.count('.') == 0:
+ token = token.replace(',', '.')
+
+ return token
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ token = self.get_token()
+ if token is None:
+ raise StopIteration
+
+ return token
+
+ def next(self):
+ return self.__next__() # Python 2.x support
+
+ @classmethod
+ def split(cls, s):
+ return list(cls(s))
+
+ @classmethod
+ def isword(cls, nextchar):
+ """ Whether or not the next character is part of a word """
+ return nextchar.isalpha()
+
+ @classmethod
+ def isnum(cls, nextchar):
+ """ Whether the next character is part of a number """
+ return nextchar.isdigit()
+
+ @classmethod
+ def isspace(cls, nextchar):
+ """ Whether the next character is whitespace """
+ return nextchar.isspace()
+
+
+class _resultbase(object):
+
+ def __init__(self):
+ for attr in self.__slots__:
+ setattr(self, attr, None)
+
+ def _repr(self, classname):
+ l = []
+ for attr in self.__slots__:
+ value = getattr(self, attr)
+ if value is not None:
+ l.append("%s=%s" % (attr, repr(value)))
+ return "%s(%s)" % (classname, ", ".join(l))
+
+ def __len__(self):
+ return (sum(getattr(self, attr) is not None
+ for attr in self.__slots__))
+
+ def __repr__(self):
+ return self._repr(self.__class__.__name__)
+
+
+class parserinfo(object):
+ """
+ Class which handles what inputs are accepted. Subclass this to customize
+ the language and acceptable values for each parameter.
+
+ :param dayfirst:
+ Whether to interpret the first value in an ambiguous 3-integer date
+ (e.g. 01/05/09) as the day (``True``) or month (``False``). If
+ ``yearfirst`` is set to ``True``, this distinguishes between YDM
+ and YMD. Default is ``False``.
+
+ :param yearfirst:
+ Whether to interpret the first value in an ambiguous 3-integer date
+ (e.g. 01/05/09) as the year. If ``True``, the first number is taken
+ to be the year, otherwise the last number is taken to be the year.
+ Default is ``False``.
+ """
+
+ # m from a.m/p.m, t from ISO T separator
+ JUMP = [" ", ".", ",", ";", "-", "/", "'",
+ "at", "on", "and", "ad", "m", "t", "of",
+ "st", "nd", "rd", "th"]
+
+ WEEKDAYS = [("Mon", "Monday"),
+ ("Tue", "Tuesday"), # TODO: "Tues"
+ ("Wed", "Wednesday"),
+ ("Thu", "Thursday"), # TODO: "Thurs"
+ ("Fri", "Friday"),
+ ("Sat", "Saturday"),
+ ("Sun", "Sunday")]
+ MONTHS = [("Jan", "January"),
+ ("Feb", "February"), # TODO: "Febr"
+ ("Mar", "March"),
+ ("Apr", "April"),
+ ("May", "May"),
+ ("Jun", "June"),
+ ("Jul", "July"),
+ ("Aug", "August"),
+ ("Sep", "Sept", "September"),
+ ("Oct", "October"),
+ ("Nov", "November"),
+ ("Dec", "December")]
+ HMS = [("h", "hour", "hours"),
+ ("m", "minute", "minutes"),
+ ("s", "second", "seconds")]
+ AMPM = [("am", "a"),
+ ("pm", "p")]
+ UTCZONE = ["UTC", "GMT", "Z", "z"]
+ PERTAIN = ["of"]
+ TZOFFSET = {}
+ # TODO: ERA = ["AD", "BC", "CE", "BCE", "Stardate",
+ # "Anno Domini", "Year of Our Lord"]
+
+ def __init__(self, dayfirst=False, yearfirst=False):
+ self._jump = self._convert(self.JUMP)
+ self._weekdays = self._convert(self.WEEKDAYS)
+ self._months = self._convert(self.MONTHS)
+ self._hms = self._convert(self.HMS)
+ self._ampm = self._convert(self.AMPM)
+ self._utczone = self._convert(self.UTCZONE)
+ self._pertain = self._convert(self.PERTAIN)
+
+ self.dayfirst = dayfirst
+ self.yearfirst = yearfirst
+
+ self._year = time.localtime().tm_year
+ self._century = self._year // 100 * 100
+
+ def _convert(self, lst):
+ dct = {}
+ for i, v in enumerate(lst):
+ if isinstance(v, tuple):
+ for v in v:
+ dct[v.lower()] = i
+ else:
+ dct[v.lower()] = i
+ return dct
+
+ def jump(self, name):
+ return name.lower() in self._jump
+
+ def weekday(self, name):
+ try:
+ return self._weekdays[name.lower()]
+ except KeyError:
+ pass
+ return None
+
+ def month(self, name):
+ try:
+ return self._months[name.lower()] + 1
+ except KeyError:
+ pass
+ return None
+
+ def hms(self, name):
+ try:
+ return self._hms[name.lower()]
+ except KeyError:
+ return None
+
+ def ampm(self, name):
+ try:
+ return self._ampm[name.lower()]
+ except KeyError:
+ return None
+
+ def pertain(self, name):
+ return name.lower() in self._pertain
+
+ def utczone(self, name):
+ return name.lower() in self._utczone
+
+ def tzoffset(self, name):
+ if name in self._utczone:
+ return 0
+
+ return self.TZOFFSET.get(name)
+
+ def convertyear(self, year, century_specified=False):
+ """
+ Converts two-digit years to year within [-50, 49]
+ range of self._year (current local time)
+ """
+
+ # Function contract is that the year is always positive
+ assert year >= 0
+
+ if year < 100 and not century_specified:
+ # assume current century to start
+ year += self._century
+
+ if year >= self._year + 50: # if too far in future
+ year -= 100
+ elif year < self._year - 50: # if too far in past
+ year += 100
+
+ return year
+
+ def validate(self, res):
+ # move to info
+ if res.year is not None:
+ res.year = self.convertyear(res.year, res.century_specified)
+
+ if ((res.tzoffset == 0 and not res.tzname) or
+ (res.tzname == 'Z' or res.tzname == 'z')):
+ res.tzname = "UTC"
+ res.tzoffset = 0
+ elif res.tzoffset != 0 and res.tzname and self.utczone(res.tzname):
+ res.tzoffset = 0
+ return True
+
+
+class _ymd(list):
+ def __init__(self, *args, **kwargs):
+ super(self.__class__, self).__init__(*args, **kwargs)
+ self.century_specified = False
+ self.dstridx = None
+ self.mstridx = None
+ self.ystridx = None
+
+ @property
+ def has_year(self):
+ return self.ystridx is not None
+
+ @property
+ def has_month(self):
+ return self.mstridx is not None
+
+ @property
+ def has_day(self):
+ return self.dstridx is not None
+
+ def could_be_day(self, value):
+ if self.has_day:
+ return False
+ elif not self.has_month:
+ return 1 <= value <= 31
+ elif not self.has_year:
+ # Be permissive, assume leap year
+ month = self[self.mstridx]
+ return 1 <= value <= monthrange(2000, month)[1]
+ else:
+ month = self[self.mstridx]
+ year = self[self.ystridx]
+ return 1 <= value <= monthrange(year, month)[1]
+
+ def append(self, val, label=None):
+ if hasattr(val, '__len__'):
+ if val.isdigit() and len(val) > 2:
+ self.century_specified = True
+ if label not in [None, 'Y']: # pragma: no cover
+ raise ValueError(label)
+ label = 'Y'
+ elif val > 100:
+ self.century_specified = True
+ if label not in [None, 'Y']: # pragma: no cover
+ raise ValueError(label)
+ label = 'Y'
+
+ super(self.__class__, self).append(int(val))
+
+ if label == 'M':
+ if self.has_month:
+ raise ValueError('Month is already set')
+ self.mstridx = len(self) - 1
+ elif label == 'D':
+ if self.has_day:
+ raise ValueError('Day is already set')
+ self.dstridx = len(self) - 1
+ elif label == 'Y':
+ if self.has_year:
+ raise ValueError('Year is already set')
+ self.ystridx = len(self) - 1
+
+ def _resolve_from_stridxs(self, strids):
+ """
+ Try to resolve the identities of year/month/day elements using
+ ystridx, mstridx, and dstridx, if enough of these are specified.
+ """
+ if len(self) == 3 and len(strids) == 2:
+ # we can back out the remaining stridx value
+ missing = [x for x in range(3) if x not in strids.values()]
+ key = [x for x in ['y', 'm', 'd'] if x not in strids]
+ assert len(missing) == len(key) == 1
+ key = key[0]
+ val = missing[0]
+ strids[key] = val
+
+ assert len(self) == len(strids) # otherwise this should not be called
+ out = {key: self[strids[key]] for key in strids}
+ return (out.get('y'), out.get('m'), out.get('d'))
+
+ def resolve_ymd(self, yearfirst, dayfirst):
+ len_ymd = len(self)
+ year, month, day = (None, None, None)
+
+ strids = (('y', self.ystridx),
+ ('m', self.mstridx),
+ ('d', self.dstridx))
+
+ strids = {key: val for key, val in strids if val is not None}
+ if (len(self) == len(strids) > 0 or
+ (len(self) == 3 and len(strids) == 2)):
+ return self._resolve_from_stridxs(strids)
+
+ mstridx = self.mstridx
+
+ if len_ymd > 3:
+ raise ValueError("More than three YMD values")
+ elif len_ymd == 1 or (mstridx is not None and len_ymd == 2):
+ # One member, or two members with a month string
+ if mstridx is not None:
+ month = self[mstridx]
+ # since mstridx is 0 or 1, self[mstridx-1] always
+ # looks up the other element
+ other = self[mstridx - 1]
+ else:
+ other = self[0]
+
+ if len_ymd > 1 or mstridx is None:
+ if other > 31:
+ year = other
+ else:
+ day = other
+
+ elif len_ymd == 2:
+ # Two members with numbers
+ if self[0] > 31:
+ # 99-01
+ year, month = self
+ elif self[1] > 31:
+ # 01-99
+ month, year = self
+ elif dayfirst and self[1] <= 12:
+ # 13-01
+ day, month = self
+ else:
+ # 01-13
+ month, day = self
+
+ elif len_ymd == 3:
+ # Three members
+ if mstridx == 0:
+ if self[1] > 31:
+ # Apr-2003-25
+ month, year, day = self
+ else:
+ month, day, year = self
+ elif mstridx == 1:
+ if self[0] > 31 or (yearfirst and self[2] <= 31):
+ # 99-Jan-01
+ year, month, day = self
+ else:
+ # 01-Jan-01
+ # Give precedence to day-first, since
+ # two-digit years is usually hand-written.
+ day, month, year = self
+
+ elif mstridx == 2:
+ # WTF!?
+ if self[1] > 31:
+ # 01-99-Jan
+ day, year, month = self
+ else:
+ # 99-01-Jan
+ year, day, month = self
+
+ else:
+ if (self[0] > 31 or
+ self.ystridx == 0 or
+ (yearfirst and self[1] <= 12 and self[2] <= 31)):
+ # 99-01-01
+ if dayfirst and self[2] <= 12:
+ year, day, month = self
+ else:
+ year, month, day = self
+ elif self[0] > 12 or (dayfirst and self[1] <= 12):
+ # 13-01-01
+ day, month, year = self
+ else:
+ # 01-13-01
+ month, day, year = self
+
+ return year, month, day
+
+
+class parser(object):
+ def __init__(self, info=None):
+ self.info = info or parserinfo()
+
+ def parse(self, timestr, default=None,
+ ignoretz=False, tzinfos=None, **kwargs):
+ """
+ Parse the date/time string into a :class:`datetime.datetime` object.
+
+ :param timestr:
+ Any date/time string using the supported formats.
+
+ :param default:
+ The default datetime object, if this is a datetime object and not
+ ``None``, elements specified in ``timestr`` replace elements in the
+ default object.
+
+ :param ignoretz:
+ If set ``True``, time zones in parsed strings are ignored and a
+ naive :class:`datetime.datetime` object is returned.
+
+ :param tzinfos:
+ Additional time zone names / aliases which may be present in the
+ string. This argument maps time zone names (and optionally offsets
+ from those time zones) to time zones. This parameter can be a
+ dictionary with timezone aliases mapping time zone names to time
+ zones or a function taking two parameters (``tzname`` and
+ ``tzoffset``) and returning a time zone.
+
+ The timezones to which the names are mapped can be an integer
+ offset from UTC in seconds or a :class:`tzinfo` object.
+
+ .. doctest::
+ :options: +NORMALIZE_WHITESPACE
+
+ >>> from dateutil.parser import parse
+ >>> from dateutil.tz import gettz
+ >>> tzinfos = {"BRST": -7200, "CST": gettz("America/Chicago")}
+ >>> parse("2012-01-19 17:21:00 BRST", tzinfos=tzinfos)
+ datetime.datetime(2012, 1, 19, 17, 21, tzinfo=tzoffset(u'BRST', -7200))
+ >>> parse("2012-01-19 17:21:00 CST", tzinfos=tzinfos)
+ datetime.datetime(2012, 1, 19, 17, 21,
+ tzinfo=tzfile('/usr/share/zoneinfo/America/Chicago'))
+
+ This parameter is ignored if ``ignoretz`` is set.
+
+ :param \\*\\*kwargs:
+ Keyword arguments as passed to ``_parse()``.
+
+ :return:
+ Returns a :class:`datetime.datetime` object or, if the
+ ``fuzzy_with_tokens`` option is ``True``, returns a tuple, the
+ first element being a :class:`datetime.datetime` object, the second
+ a tuple containing the fuzzy tokens.
+
+ :raises ParserError:
+ Raised for invalid or unknown string format, if the provided
+ :class:`tzinfo` is not in a valid format, or if an invalid date
+ would be created.
+
+ :raises TypeError:
+ Raised for non-string or character stream input.
+
+ :raises OverflowError:
+ Raised if the parsed date exceeds the largest valid C integer on
+ your system.
+ """
+
+ if default is None:
+ default = datetime.datetime.now().replace(hour=0, minute=0,
+ second=0, microsecond=0)
+
+ res, skipped_tokens = self._parse(timestr, **kwargs)
+
+ if res is None:
+ raise ParserError("Unknown string format: %s", timestr)
+
+ if len(res) == 0:
+ raise ParserError("String does not contain a date: %s", timestr)
+
+ try:
+ ret = self._build_naive(res, default)
+ except ValueError as e:
+ six.raise_from(ParserError(str(e) + ": %s", timestr), e)
+
+ if not ignoretz:
+ ret = self._build_tzaware(ret, res, tzinfos)
+
+ if kwargs.get('fuzzy_with_tokens', False):
+ return ret, skipped_tokens
+ else:
+ return ret
+
+ class _result(_resultbase):
+ __slots__ = ["year", "month", "day", "weekday",
+ "hour", "minute", "second", "microsecond",
+ "tzname", "tzoffset", "ampm","any_unused_tokens"]
+
+ def _parse(self, timestr, dayfirst=None, yearfirst=None, fuzzy=False,
+ fuzzy_with_tokens=False):
+ """
+ Private method which performs the heavy lifting of parsing, called from
+ ``parse()``, which passes on its ``kwargs`` to this function.
+
+ :param timestr:
+ The string to parse.
+
+ :param dayfirst:
+ Whether to interpret the first value in an ambiguous 3-integer date
+ (e.g. 01/05/09) as the day (``True``) or month (``False``). If
+ ``yearfirst`` is set to ``True``, this distinguishes between YDM
+ and YMD. If set to ``None``, this value is retrieved from the
+ current :class:`parserinfo` object (which itself defaults to
+ ``False``).
+
+ :param yearfirst:
+ Whether to interpret the first value in an ambiguous 3-integer date
+ (e.g. 01/05/09) as the year. If ``True``, the first number is taken
+ to be the year, otherwise the last number is taken to be the year.
+ If this is set to ``None``, the value is retrieved from the current
+ :class:`parserinfo` object (which itself defaults to ``False``).
+
+ :param fuzzy:
+ Whether to allow fuzzy parsing, allowing for string like "Today is
+ January 1, 2047 at 8:21:00AM".
+
+ :param fuzzy_with_tokens:
+ If ``True``, ``fuzzy`` is automatically set to True, and the parser
+ will return a tuple where the first element is the parsed
+ :class:`datetime.datetime` datetimestamp and the second element is
+ a tuple containing the portions of the string which were ignored:
+
+ .. doctest::
+
+ >>> from dateutil.parser import parse
+ >>> parse("Today is January 1, 2047 at 8:21:00AM", fuzzy_with_tokens=True)
+ (datetime.datetime(2047, 1, 1, 8, 21), (u'Today is ', u' ', u'at '))
+
+ """
+ if fuzzy_with_tokens:
+ fuzzy = True
+
+ info = self.info
+
+ if dayfirst is None:
+ dayfirst = info.dayfirst
+
+ if yearfirst is None:
+ yearfirst = info.yearfirst
+
+ res = self._result()
+ l = _timelex.split(timestr) # Splits the timestr into tokens
+
+ skipped_idxs = []
+
+ # year/month/day list
+ ymd = _ymd()
+
+ len_l = len(l)
+ i = 0
+ try:
+ while i < len_l:
+
+ # Check if it's a number
+ value_repr = l[i]
+ try:
+ value = float(value_repr)
+ except ValueError:
+ value = None
+
+ if value is not None:
+ # Numeric token
+ i = self._parse_numeric_token(l, i, info, ymd, res, fuzzy)
+
+ # Check weekday
+ elif info.weekday(l[i]) is not None:
+ value = info.weekday(l[i])
+ res.weekday = value
+
+ # Check month name
+ elif info.month(l[i]) is not None:
+ value = info.month(l[i])
+ ymd.append(value, 'M')
+
+ if i + 1 < len_l:
+ if l[i + 1] in ('-', '/'):
+ # Jan-01[-99]
+ sep = l[i + 1]
+ ymd.append(l[i + 2])
+
+ if i + 3 < len_l and l[i + 3] == sep:
+ # Jan-01-99
+ ymd.append(l[i + 4])
+ i += 2
+
+ i += 2
+
+ elif (i + 4 < len_l and l[i + 1] == l[i + 3] == ' ' and
+ info.pertain(l[i + 2])):
+ # Jan of 01
+ # In this case, 01 is clearly year
+ if l[i + 4].isdigit():
+ # Convert it here to become unambiguous
+ value = int(l[i + 4])
+ year = str(info.convertyear(value))
+ ymd.append(year, 'Y')
+ else:
+ # Wrong guess
+ pass
+ # TODO: not hit in tests
+ i += 4
+
+ # Check am/pm
+ elif info.ampm(l[i]) is not None:
+ value = info.ampm(l[i])
+ val_is_ampm = self._ampm_valid(res.hour, res.ampm, fuzzy)
+
+ if val_is_ampm:
+ res.hour = self._adjust_ampm(res.hour, value)
+ res.ampm = value
+
+ elif fuzzy:
+ skipped_idxs.append(i)
+
+ # Check for a timezone name
+ elif self._could_be_tzname(res.hour, res.tzname, res.tzoffset, l[i]):
+ res.tzname = l[i]
+ res.tzoffset = info.tzoffset(res.tzname)
+
+ # Check for something like GMT+3, or BRST+3. Notice
+ # that it doesn't mean "I am 3 hours after GMT", but
+ # "my time +3 is GMT". If found, we reverse the
+ # logic so that timezone parsing code will get it
+ # right.
+ if i + 1 < len_l and l[i + 1] in ('+', '-'):
+ l[i + 1] = ('+', '-')[l[i + 1] == '+']
+ res.tzoffset = None
+ if info.utczone(res.tzname):
+ # With something like GMT+3, the timezone
+ # is *not* GMT.
+ res.tzname = None
+
+ # Check for a numbered timezone
+ elif res.hour is not None and l[i] in ('+', '-'):
+ signal = (-1, 1)[l[i] == '+']
+ len_li = len(l[i + 1])
+
+ # TODO: check that l[i + 1] is integer?
+ if len_li == 4:
+ # -0300
+ hour_offset = int(l[i + 1][:2])
+ min_offset = int(l[i + 1][2:])
+ elif i + 2 < len_l and l[i + 2] == ':':
+ # -03:00
+ hour_offset = int(l[i + 1])
+ min_offset = int(l[i + 3]) # TODO: Check that l[i+3] is minute-like?
+ i += 2
+ elif len_li <= 2:
+ # -[0]3
+ hour_offset = int(l[i + 1][:2])
+ min_offset = 0
+ else:
+ raise ValueError(timestr)
+
+ res.tzoffset = signal * (hour_offset * 3600 + min_offset * 60)
+
+ # Look for a timezone name between parenthesis
+ if (i + 5 < len_l and
+ info.jump(l[i + 2]) and l[i + 3] == '(' and
+ l[i + 5] == ')' and
+ 3 <= len(l[i + 4]) and
+ self._could_be_tzname(res.hour, res.tzname,
+ None, l[i + 4])):
+ # -0300 (BRST)
+ res.tzname = l[i + 4]
+ i += 4
+
+ i += 1
+
+ # Check jumps
+ elif not (info.jump(l[i]) or fuzzy):
+ raise ValueError(timestr)
+
+ else:
+ skipped_idxs.append(i)
+ i += 1
+
+ # Process year/month/day
+ year, month, day = ymd.resolve_ymd(yearfirst, dayfirst)
+
+ res.century_specified = ymd.century_specified
+ res.year = year
+ res.month = month
+ res.day = day
+
+ except (IndexError, ValueError):
+ return None, None
+
+ if not info.validate(res):
+ return None, None
+
+ if fuzzy_with_tokens:
+ skipped_tokens = self._recombine_skipped(l, skipped_idxs)
+ return res, tuple(skipped_tokens)
+ else:
+ return res, None
+
+ def _parse_numeric_token(self, tokens, idx, info, ymd, res, fuzzy):
+ # Token is a number
+ value_repr = tokens[idx]
+ try:
+ value = self._to_decimal(value_repr)
+ except Exception as e:
+ six.raise_from(ValueError('Unknown numeric token'), e)
+
+ len_li = len(value_repr)
+
+ len_l = len(tokens)
+
+ if (len(ymd) == 3 and len_li in (2, 4) and
+ res.hour is None and
+ (idx + 1 >= len_l or
+ (tokens[idx + 1] != ':' and
+ info.hms(tokens[idx + 1]) is None))):
+ # 19990101T23[59]
+ s = tokens[idx]
+ res.hour = int(s[:2])
+
+ if len_li == 4:
+ res.minute = int(s[2:])
+
+ elif len_li == 6 or (len_li > 6 and tokens[idx].find('.') == 6):
+ # YYMMDD or HHMMSS[.ss]
+ s = tokens[idx]
+
+ if not ymd and '.' not in tokens[idx]:
+ ymd.append(s[:2])
+ ymd.append(s[2:4])
+ ymd.append(s[4:])
+ else:
+ # 19990101T235959[.59]
+
+ # TODO: Check if res attributes already set.
+ res.hour = int(s[:2])
+ res.minute = int(s[2:4])
+ res.second, res.microsecond = self._parsems(s[4:])
+
+ elif len_li in (8, 12, 14):
+ # YYYYMMDD
+ s = tokens[idx]
+ ymd.append(s[:4], 'Y')
+ ymd.append(s[4:6])
+ ymd.append(s[6:8])
+
+ if len_li > 8:
+ res.hour = int(s[8:10])
+ res.minute = int(s[10:12])
+
+ if len_li > 12:
+ res.second = int(s[12:])
+
+ elif self._find_hms_idx(idx, tokens, info, allow_jump=True) is not None:
+ # HH[ ]h or MM[ ]m or SS[.ss][ ]s
+ hms_idx = self._find_hms_idx(idx, tokens, info, allow_jump=True)
+ (idx, hms) = self._parse_hms(idx, tokens, info, hms_idx)
+ if hms is not None:
+ # TODO: checking that hour/minute/second are not
+ # already set?
+ self._assign_hms(res, value_repr, hms)
+
+ elif idx + 2 < len_l and tokens[idx + 1] == ':':
+ # HH:MM[:SS[.ss]]
+ res.hour = int(value)
+ value = self._to_decimal(tokens[idx + 2]) # TODO: try/except for this?
+ (res.minute, res.second) = self._parse_min_sec(value)
+
+ if idx + 4 < len_l and tokens[idx + 3] == ':':
+ res.second, res.microsecond = self._parsems(tokens[idx + 4])
+
+ idx += 2
+
+ idx += 2
+
+ elif idx + 1 < len_l and tokens[idx + 1] in ('-', '/', '.'):
+ sep = tokens[idx + 1]
+ ymd.append(value_repr)
+
+ if idx + 2 < len_l and not info.jump(tokens[idx + 2]):
+ if tokens[idx + 2].isdigit():
+ # 01-01[-01]
+ ymd.append(tokens[idx + 2])
+ else:
+ # 01-Jan[-01]
+ value = info.month(tokens[idx + 2])
+
+ if value is not None:
+ ymd.append(value, 'M')
+ else:
+ raise ValueError()
+
+ if idx + 3 < len_l and tokens[idx + 3] == sep:
+ # We have three members
+ value = info.month(tokens[idx + 4])
+
+ if value is not None:
+ ymd.append(value, 'M')
+ else:
+ ymd.append(tokens[idx + 4])
+ idx += 2
+
+ idx += 1
+ idx += 1
+
+ elif idx + 1 >= len_l or info.jump(tokens[idx + 1]):
+ if idx + 2 < len_l and info.ampm(tokens[idx + 2]) is not None:
+ # 12 am
+ hour = int(value)
+ res.hour = self._adjust_ampm(hour, info.ampm(tokens[idx + 2]))
+ idx += 1
+ else:
+ # Year, month or day
+ ymd.append(value)
+ idx += 1
+
+ elif info.ampm(tokens[idx + 1]) is not None and (0 <= value < 24):
+ # 12am
+ hour = int(value)
+ res.hour = self._adjust_ampm(hour, info.ampm(tokens[idx + 1]))
+ idx += 1
+
+ elif ymd.could_be_day(value):
+ ymd.append(value)
+
+ elif not fuzzy:
+ raise ValueError()
+
+ return idx
+
+ def _find_hms_idx(self, idx, tokens, info, allow_jump):
+ len_l = len(tokens)
+
+ if idx+1 < len_l and info.hms(tokens[idx+1]) is not None:
+ # There is an "h", "m", or "s" label following this token. We take
+ # assign the upcoming label to the current token.
+ # e.g. the "12" in 12h"
+ hms_idx = idx + 1
+
+ elif (allow_jump and idx+2 < len_l and tokens[idx+1] == ' ' and
+ info.hms(tokens[idx+2]) is not None):
+ # There is a space and then an "h", "m", or "s" label.
+ # e.g. the "12" in "12 h"
+ hms_idx = idx + 2
+
+ elif idx > 0 and info.hms(tokens[idx-1]) is not None:
+ # There is a "h", "m", or "s" preceding this token. Since neither
+ # of the previous cases was hit, there is no label following this
+ # token, so we use the previous label.
+ # e.g. the "04" in "12h04"
+ hms_idx = idx-1
+
+ elif (1 < idx == len_l-1 and tokens[idx-1] == ' ' and
+ info.hms(tokens[idx-2]) is not None):
+ # If we are looking at the final token, we allow for a
+ # backward-looking check to skip over a space.
+ # TODO: Are we sure this is the right condition here?
+ hms_idx = idx - 2
+
+ else:
+ hms_idx = None
+
+ return hms_idx
+
+ def _assign_hms(self, res, value_repr, hms):
+ # See GH issue #427, fixing float rounding
+ value = self._to_decimal(value_repr)
+
+ if hms == 0:
+ # Hour
+ res.hour = int(value)
+ if value % 1:
+ res.minute = int(60*(value % 1))
+
+ elif hms == 1:
+ (res.minute, res.second) = self._parse_min_sec(value)
+
+ elif hms == 2:
+ (res.second, res.microsecond) = self._parsems(value_repr)
+
+ def _could_be_tzname(self, hour, tzname, tzoffset, token):
+ return (hour is not None and
+ tzname is None and
+ tzoffset is None and
+ len(token) <= 5 and
+ (all(x in string.ascii_uppercase for x in token)
+ or token in self.info.UTCZONE))
+
+ def _ampm_valid(self, hour, ampm, fuzzy):
+ """
+ For fuzzy parsing, 'a' or 'am' (both valid English words)
+ may erroneously trigger the AM/PM flag. Deal with that
+ here.
+ """
+ val_is_ampm = True
+
+ # If there's already an AM/PM flag, this one isn't one.
+ if fuzzy and ampm is not None:
+ val_is_ampm = False
+
+ # If AM/PM is found and hour is not, raise a ValueError
+ if hour is None:
+ if fuzzy:
+ val_is_ampm = False
+ else:
+ raise ValueError('No hour specified with AM or PM flag.')
+ elif not 0 <= hour <= 12:
+ # If AM/PM is found, it's a 12 hour clock, so raise
+ # an error for invalid range
+ if fuzzy:
+ val_is_ampm = False
+ else:
+ raise ValueError('Invalid hour specified for 12-hour clock.')
+
+ return val_is_ampm
+
+ def _adjust_ampm(self, hour, ampm):
+ if hour < 12 and ampm == 1:
+ hour += 12
+ elif hour == 12 and ampm == 0:
+ hour = 0
+ return hour
+
+ def _parse_min_sec(self, value):
+ # TODO: Every usage of this function sets res.second to the return
+ # value. Are there any cases where second will be returned as None and
+ # we *don't* want to set res.second = None?
+ minute = int(value)
+ second = None
+
+ sec_remainder = value % 1
+ if sec_remainder:
+ second = int(60 * sec_remainder)
+ return (minute, second)
+
+ def _parse_hms(self, idx, tokens, info, hms_idx):
+ # TODO: Is this going to admit a lot of false-positives for when we
+ # just happen to have digits and "h", "m" or "s" characters in non-date
+ # text? I guess hex hashes won't have that problem, but there's plenty
+ # of random junk out there.
+ if hms_idx is None:
+ hms = None
+ new_idx = idx
+ elif hms_idx > idx:
+ hms = info.hms(tokens[hms_idx])
+ new_idx = hms_idx
+ else:
+ # Looking backwards, increment one.
+ hms = info.hms(tokens[hms_idx]) + 1
+ new_idx = idx
+
+ return (new_idx, hms)
+
+ # ------------------------------------------------------------------
+ # Handling for individual tokens. These are kept as methods instead
+ # of functions for the sake of customizability via subclassing.
+
+ def _parsems(self, value):
+ """Parse a I[.F] seconds value into (seconds, microseconds)."""
+ if "." not in value:
+ return int(value), 0
+ else:
+ i, f = value.split(".")
+ return int(i), int(f.ljust(6, "0")[:6])
+
+ def _to_decimal(self, val):
+ try:
+ decimal_value = Decimal(val)
+ # See GH 662, edge case, infinite value should not be converted
+ # via `_to_decimal`
+ if not decimal_value.is_finite():
+ raise ValueError("Converted decimal value is infinite or NaN")
+ except Exception as e:
+ msg = "Could not convert %s to decimal" % val
+ six.raise_from(ValueError(msg), e)
+ else:
+ return decimal_value
+
+ # ------------------------------------------------------------------
+ # Post-Parsing construction of datetime output. These are kept as
+ # methods instead of functions for the sake of customizability via
+ # subclassing.
+
+ def _build_tzinfo(self, tzinfos, tzname, tzoffset):
+ if callable(tzinfos):
+ tzdata = tzinfos(tzname, tzoffset)
+ else:
+ tzdata = tzinfos.get(tzname)
+ # handle case where tzinfo is paased an options that returns None
+ # eg tzinfos = {'BRST' : None}
+ if isinstance(tzdata, datetime.tzinfo) or tzdata is None:
+ tzinfo = tzdata
+ elif isinstance(tzdata, text_type):
+ tzinfo = tz.tzstr(tzdata)
+ elif isinstance(tzdata, integer_types):
+ tzinfo = tz.tzoffset(tzname, tzdata)
+ else:
+ raise TypeError("Offset must be tzinfo subclass, tz string, "
+ "or int offset.")
+ return tzinfo
+
+ def _build_tzaware(self, naive, res, tzinfos):
+ if (callable(tzinfos) or (tzinfos and res.tzname in tzinfos)):
+ tzinfo = self._build_tzinfo(tzinfos, res.tzname, res.tzoffset)
+ aware = naive.replace(tzinfo=tzinfo)
+ aware = self._assign_tzname(aware, res.tzname)
+
+ elif res.tzname and res.tzname in time.tzname:
+ aware = naive.replace(tzinfo=tz.tzlocal())
+
+ # Handle ambiguous local datetime
+ aware = self._assign_tzname(aware, res.tzname)
+
+ # This is mostly relevant for winter GMT zones parsed in the UK
+ if (aware.tzname() != res.tzname and
+ res.tzname in self.info.UTCZONE):
+ aware = aware.replace(tzinfo=tz.UTC)
+
+ elif res.tzoffset == 0:
+ aware = naive.replace(tzinfo=tz.UTC)
+
+ elif res.tzoffset:
+ aware = naive.replace(tzinfo=tz.tzoffset(res.tzname, res.tzoffset))
+
+ elif not res.tzname and not res.tzoffset:
+ # i.e. no timezone information was found.
+ aware = naive
+
+ elif res.tzname:
+ # tz-like string was parsed but we don't know what to do
+ # with it
+ warnings.warn("tzname {tzname} identified but not understood. "
+ "Pass `tzinfos` argument in order to correctly "
+ "return a timezone-aware datetime. In a future "
+ "version, this will raise an "
+ "exception.".format(tzname=res.tzname),
+ category=UnknownTimezoneWarning)
+ aware = naive
+
+ return aware
+
+ def _build_naive(self, res, default):
+ repl = {}
+ for attr in ("year", "month", "day", "hour",
+ "minute", "second", "microsecond"):
+ value = getattr(res, attr)
+ if value is not None:
+ repl[attr] = value
+
+ if 'day' not in repl:
+ # If the default day exceeds the last day of the month, fall back
+ # to the end of the month.
+ cyear = default.year if res.year is None else res.year
+ cmonth = default.month if res.month is None else res.month
+ cday = default.day if res.day is None else res.day
+
+ if cday > monthrange(cyear, cmonth)[1]:
+ repl['day'] = monthrange(cyear, cmonth)[1]
+
+ naive = default.replace(**repl)
+
+ if res.weekday is not None and not res.day:
+ naive = naive + relativedelta.relativedelta(weekday=res.weekday)
+
+ return naive
+
+ def _assign_tzname(self, dt, tzname):
+ if dt.tzname() != tzname:
+ new_dt = tz.enfold(dt, fold=1)
+ if new_dt.tzname() == tzname:
+ return new_dt
+
+ return dt
+
+ def _recombine_skipped(self, tokens, skipped_idxs):
+ """
+ >>> tokens = ["foo", " ", "bar", " ", "19June2000", "baz"]
+ >>> skipped_idxs = [0, 1, 2, 5]
+ >>> _recombine_skipped(tokens, skipped_idxs)
+ ["foo bar", "baz"]
+ """
+ skipped_tokens = []
+ for i, idx in enumerate(sorted(skipped_idxs)):
+ if i > 0 and idx - 1 == skipped_idxs[i - 1]:
+ skipped_tokens[-1] = skipped_tokens[-1] + tokens[idx]
+ else:
+ skipped_tokens.append(tokens[idx])
+
+ return skipped_tokens
+
+
+DEFAULTPARSER = parser()
+
+
+def parse(timestr, parserinfo=None, **kwargs):
+ """
+
+ Parse a string in one of the supported formats, using the
+ ``parserinfo`` parameters.
+
+ :param timestr:
+ A string containing a date/time stamp.
+
+ :param parserinfo:
+ A :class:`parserinfo` object containing parameters for the parser.
+ If ``None``, the default arguments to the :class:`parserinfo`
+ constructor are used.
+
+ The ``**kwargs`` parameter takes the following keyword arguments:
+
+ :param default:
+ The default datetime object, if this is a datetime object and not
+ ``None``, elements specified in ``timestr`` replace elements in the
+ default object.
+
+ :param ignoretz:
+ If set ``True``, time zones in parsed strings are ignored and a naive
+ :class:`datetime` object is returned.
+
+ :param tzinfos:
+ Additional time zone names / aliases which may be present in the
+ string. This argument maps time zone names (and optionally offsets
+ from those time zones) to time zones. This parameter can be a
+ dictionary with timezone aliases mapping time zone names to time
+ zones or a function taking two parameters (``tzname`` and
+ ``tzoffset``) and returning a time zone.
+
+ The timezones to which the names are mapped can be an integer
+ offset from UTC in seconds or a :class:`tzinfo` object.
+
+ .. doctest::
+ :options: +NORMALIZE_WHITESPACE
+
+ >>> from dateutil.parser import parse
+ >>> from dateutil.tz import gettz
+ >>> tzinfos = {"BRST": -7200, "CST": gettz("America/Chicago")}
+ >>> parse("2012-01-19 17:21:00 BRST", tzinfos=tzinfos)
+ datetime.datetime(2012, 1, 19, 17, 21, tzinfo=tzoffset(u'BRST', -7200))
+ >>> parse("2012-01-19 17:21:00 CST", tzinfos=tzinfos)
+ datetime.datetime(2012, 1, 19, 17, 21,
+ tzinfo=tzfile('/usr/share/zoneinfo/America/Chicago'))
+
+ This parameter is ignored if ``ignoretz`` is set.
+
+ :param dayfirst:
+ Whether to interpret the first value in an ambiguous 3-integer date
+ (e.g. 01/05/09) as the day (``True``) or month (``False``). If
+ ``yearfirst`` is set to ``True``, this distinguishes between YDM and
+ YMD. If set to ``None``, this value is retrieved from the current
+ :class:`parserinfo` object (which itself defaults to ``False``).
+
+ :param yearfirst:
+ Whether to interpret the first value in an ambiguous 3-integer date
+ (e.g. 01/05/09) as the year. If ``True``, the first number is taken to
+ be the year, otherwise the last number is taken to be the year. If
+ this is set to ``None``, the value is retrieved from the current
+ :class:`parserinfo` object (which itself defaults to ``False``).
+
+ :param fuzzy:
+ Whether to allow fuzzy parsing, allowing for string like "Today is
+ January 1, 2047 at 8:21:00AM".
+
+ :param fuzzy_with_tokens:
+ If ``True``, ``fuzzy`` is automatically set to True, and the parser
+ will return a tuple where the first element is the parsed
+ :class:`datetime.datetime` datetimestamp and the second element is
+ a tuple containing the portions of the string which were ignored:
+
+ .. doctest::
+
+ >>> from dateutil.parser import parse
+ >>> parse("Today is January 1, 2047 at 8:21:00AM", fuzzy_with_tokens=True)
+ (datetime.datetime(2047, 1, 1, 8, 21), (u'Today is ', u' ', u'at '))
+
+ :return:
+ Returns a :class:`datetime.datetime` object or, if the
+ ``fuzzy_with_tokens`` option is ``True``, returns a tuple, the
+ first element being a :class:`datetime.datetime` object, the second
+ a tuple containing the fuzzy tokens.
+
+ :raises ParserError:
+ Raised for invalid or unknown string formats, if the provided
+ :class:`tzinfo` is not in a valid format, or if an invalid date would
+ be created.
+
+ :raises OverflowError:
+ Raised if the parsed date exceeds the largest valid C integer on
+ your system.
+ """
+ if parserinfo:
+ return parser(parserinfo).parse(timestr, **kwargs)
+ else:
+ return DEFAULTPARSER.parse(timestr, **kwargs)
+
+
+class _tzparser(object):
+
+ class _result(_resultbase):
+
+ __slots__ = ["stdabbr", "stdoffset", "dstabbr", "dstoffset",
+ "start", "end"]
+
+ class _attr(_resultbase):
+ __slots__ = ["month", "week", "weekday",
+ "yday", "jyday", "day", "time"]
+
+ def __repr__(self):
+ return self._repr("")
+
+ def __init__(self):
+ _resultbase.__init__(self)
+ self.start = self._attr()
+ self.end = self._attr()
+
+ def parse(self, tzstr):
+ res = self._result()
+ l = [x for x in re.split(r'([,:.]|[a-zA-Z]+|[0-9]+)',tzstr) if x]
+ used_idxs = list()
+ try:
+
+ len_l = len(l)
+
+ i = 0
+ while i < len_l:
+ # BRST+3[BRDT[+2]]
+ j = i
+ while j < len_l and not [x for x in l[j]
+ if x in "0123456789:,-+"]:
+ j += 1
+ if j != i:
+ if not res.stdabbr:
+ offattr = "stdoffset"
+ res.stdabbr = "".join(l[i:j])
+ else:
+ offattr = "dstoffset"
+ res.dstabbr = "".join(l[i:j])
+
+ for ii in range(j):
+ used_idxs.append(ii)
+ i = j
+ if (i < len_l and (l[i] in ('+', '-') or l[i][0] in
+ "0123456789")):
+ if l[i] in ('+', '-'):
+ # Yes, that's right. See the TZ variable
+ # documentation.
+ signal = (1, -1)[l[i] == '+']
+ used_idxs.append(i)
+ i += 1
+ else:
+ signal = -1
+ len_li = len(l[i])
+ if len_li == 4:
+ # -0300
+ setattr(res, offattr, (int(l[i][:2]) * 3600 +
+ int(l[i][2:]) * 60) * signal)
+ elif i + 1 < len_l and l[i + 1] == ':':
+ # -03:00
+ setattr(res, offattr,
+ (int(l[i]) * 3600 +
+ int(l[i + 2]) * 60) * signal)
+ used_idxs.append(i)
+ i += 2
+ elif len_li <= 2:
+ # -[0]3
+ setattr(res, offattr,
+ int(l[i][:2]) * 3600 * signal)
+ else:
+ return None
+ used_idxs.append(i)
+ i += 1
+ if res.dstabbr:
+ break
+ else:
+ break
+
+
+ if i < len_l:
+ for j in range(i, len_l):
+ if l[j] == ';':
+ l[j] = ','
+
+ assert l[i] == ','
+
+ i += 1
+
+ if i >= len_l:
+ pass
+ elif (8 <= l.count(',') <= 9 and
+ not [y for x in l[i:] if x != ','
+ for y in x if y not in "0123456789+-"]):
+ # GMT0BST,3,0,30,3600,10,0,26,7200[,3600]
+ for x in (res.start, res.end):
+ x.month = int(l[i])
+ used_idxs.append(i)
+ i += 2
+ if l[i] == '-':
+ value = int(l[i + 1]) * -1
+ used_idxs.append(i)
+ i += 1
+ else:
+ value = int(l[i])
+ used_idxs.append(i)
+ i += 2
+ if value:
+ x.week = value
+ x.weekday = (int(l[i]) - 1) % 7
+ else:
+ x.day = int(l[i])
+ used_idxs.append(i)
+ i += 2
+ x.time = int(l[i])
+ used_idxs.append(i)
+ i += 2
+ if i < len_l:
+ if l[i] in ('-', '+'):
+ signal = (-1, 1)[l[i] == "+"]
+ used_idxs.append(i)
+ i += 1
+ else:
+ signal = 1
+ used_idxs.append(i)
+ res.dstoffset = (res.stdoffset + int(l[i]) * signal)
+
+ # This was a made-up format that is not in normal use
+ warn(('Parsed time zone "%s"' % tzstr) +
+ 'is in a non-standard dateutil-specific format, which ' +
+ 'is now deprecated; support for parsing this format ' +
+ 'will be removed in future versions. It is recommended ' +
+ 'that you switch to a standard format like the GNU ' +
+ 'TZ variable format.', tz.DeprecatedTzFormatWarning)
+ elif (l.count(',') == 2 and l[i:].count('/') <= 2 and
+ not [y for x in l[i:] if x not in (',', '/', 'J', 'M',
+ '.', '-', ':')
+ for y in x if y not in "0123456789"]):
+ for x in (res.start, res.end):
+ if l[i] == 'J':
+ # non-leap year day (1 based)
+ used_idxs.append(i)
+ i += 1
+ x.jyday = int(l[i])
+ elif l[i] == 'M':
+ # month[-.]week[-.]weekday
+ used_idxs.append(i)
+ i += 1
+ x.month = int(l[i])
+ used_idxs.append(i)
+ i += 1
+ assert l[i] in ('-', '.')
+ used_idxs.append(i)
+ i += 1
+ x.week = int(l[i])
+ if x.week == 5:
+ x.week = -1
+ used_idxs.append(i)
+ i += 1
+ assert l[i] in ('-', '.')
+ used_idxs.append(i)
+ i += 1
+ x.weekday = (int(l[i]) - 1) % 7
+ else:
+ # year day (zero based)
+ x.yday = int(l[i]) + 1
+
+ used_idxs.append(i)
+ i += 1
+
+ if i < len_l and l[i] == '/':
+ used_idxs.append(i)
+ i += 1
+ # start time
+ len_li = len(l[i])
+ if len_li == 4:
+ # -0300
+ x.time = (int(l[i][:2]) * 3600 +
+ int(l[i][2:]) * 60)
+ elif i + 1 < len_l and l[i + 1] == ':':
+ # -03:00
+ x.time = int(l[i]) * 3600 + int(l[i + 2]) * 60
+ used_idxs.append(i)
+ i += 2
+ if i + 1 < len_l and l[i + 1] == ':':
+ used_idxs.append(i)
+ i += 2
+ x.time += int(l[i])
+ elif len_li <= 2:
+ # -[0]3
+ x.time = (int(l[i][:2]) * 3600)
+ else:
+ return None
+ used_idxs.append(i)
+ i += 1
+
+ assert i == len_l or l[i] == ','
+
+ i += 1
+
+ assert i >= len_l
+
+ except (IndexError, ValueError, AssertionError):
+ return None
+
+ unused_idxs = set(range(len_l)).difference(used_idxs)
+ res.any_unused_tokens = not {l[n] for n in unused_idxs}.issubset({",",":"})
+ return res
+
+
+DEFAULTTZPARSER = _tzparser()
+
+
+def _parsetz(tzstr):
+ return DEFAULTTZPARSER.parse(tzstr)
+
+
+class ParserError(ValueError):
+ """Exception subclass used for any failure to parse a datetime string.
+
+ This is a subclass of :py:exc:`ValueError`, and should be raised any time
+ earlier versions of ``dateutil`` would have raised ``ValueError``.
+
+ .. versionadded:: 2.8.1
+ """
+ def __str__(self):
+ try:
+ return self.args[0] % self.args[1:]
+ except (TypeError, IndexError):
+ return super(ParserError, self).__str__()
+
+ def __repr__(self):
+ args = ", ".join("'%s'" % arg for arg in self.args)
+ return "%s(%s)" % (self.__class__.__name__, args)
+
+
+class UnknownTimezoneWarning(RuntimeWarning):
+ """Raised when the parser finds a timezone it cannot parse into a tzinfo.
+
+ .. versionadded:: 2.7.0
+ """
+# vim:ts=4:sw=4:et
diff --git a/venv/Lib/site-packages/dateutil/parser/isoparser.py b/venv/Lib/site-packages/dateutil/parser/isoparser.py
new file mode 100644
index 0000000000000000000000000000000000000000..7060087df4776a07347cbb60127a70db393e3a65
--- /dev/null
+++ b/venv/Lib/site-packages/dateutil/parser/isoparser.py
@@ -0,0 +1,416 @@
+# -*- coding: utf-8 -*-
+"""
+This module offers a parser for ISO-8601 strings
+
+It is intended to support all valid date, time and datetime formats per the
+ISO-8601 specification.
+
+..versionadded:: 2.7.0
+"""
+from datetime import datetime, timedelta, time, date
+import calendar
+from dateutil import tz
+
+from functools import wraps
+
+import re
+import six
+
+__all__ = ["isoparse", "isoparser"]
+
+
+def _takes_ascii(f):
+ @wraps(f)
+ def func(self, str_in, *args, **kwargs):
+ # If it's a stream, read the whole thing
+ str_in = getattr(str_in, 'read', lambda: str_in)()
+
+ # If it's unicode, turn it into bytes, since ISO-8601 only covers ASCII
+ if isinstance(str_in, six.text_type):
+ # ASCII is the same in UTF-8
+ try:
+ str_in = str_in.encode('ascii')
+ except UnicodeEncodeError as e:
+ msg = 'ISO-8601 strings should contain only ASCII characters'
+ six.raise_from(ValueError(msg), e)
+
+ return f(self, str_in, *args, **kwargs)
+
+ return func
+
+
+class isoparser(object):
+ def __init__(self, sep=None):
+ """
+ :param sep:
+ A single character that separates date and time portions. If
+ ``None``, the parser will accept any single character.
+ For strict ISO-8601 adherence, pass ``'T'``.
+ """
+ if sep is not None:
+ if (len(sep) != 1 or ord(sep) >= 128 or sep in '0123456789'):
+ raise ValueError('Separator must be a single, non-numeric ' +
+ 'ASCII character')
+
+ sep = sep.encode('ascii')
+
+ self._sep = sep
+
+ @_takes_ascii
+ def isoparse(self, dt_str):
+ """
+ Parse an ISO-8601 datetime string into a :class:`datetime.datetime`.
+
+ An ISO-8601 datetime string consists of a date portion, followed
+ optionally by a time portion - the date and time portions are separated
+ by a single character separator, which is ``T`` in the official
+ standard. Incomplete date formats (such as ``YYYY-MM``) may *not* be
+ combined with a time portion.
+
+ Supported date formats are:
+
+ Common:
+
+ - ``YYYY``
+ - ``YYYY-MM``
+ - ``YYYY-MM-DD`` or ``YYYYMMDD``
+
+ Uncommon:
+
+ - ``YYYY-Www`` or ``YYYYWww`` - ISO week (day defaults to 0)
+ - ``YYYY-Www-D`` or ``YYYYWwwD`` - ISO week and day
+
+ The ISO week and day numbering follows the same logic as
+ :func:`datetime.date.isocalendar`.
+
+ Supported time formats are:
+
+ - ``hh``
+ - ``hh:mm`` or ``hhmm``
+ - ``hh:mm:ss`` or ``hhmmss``
+ - ``hh:mm:ss.ssssss`` (Up to 6 sub-second digits)
+
+ Midnight is a special case for `hh`, as the standard supports both
+ 00:00 and 24:00 as a representation. The decimal separator can be
+ either a dot or a comma.
+
+
+ .. caution::
+
+ Support for fractional components other than seconds is part of the
+ ISO-8601 standard, but is not currently implemented in this parser.
+
+ Supported time zone offset formats are:
+
+ - `Z` (UTC)
+ - `±HH:MM`
+ - `±HHMM`
+ - `±HH`
+
+ Offsets will be represented as :class:`dateutil.tz.tzoffset` objects,
+ with the exception of UTC, which will be represented as
+ :class:`dateutil.tz.tzutc`. Time zone offsets equivalent to UTC (such
+ as `+00:00`) will also be represented as :class:`dateutil.tz.tzutc`.
+
+ :param dt_str:
+ A string or stream containing only an ISO-8601 datetime string
+
+ :return:
+ Returns a :class:`datetime.datetime` representing the string.
+ Unspecified components default to their lowest value.
+
+ .. warning::
+
+ As of version 2.7.0, the strictness of the parser should not be
+ considered a stable part of the contract. Any valid ISO-8601 string
+ that parses correctly with the default settings will continue to
+ parse correctly in future versions, but invalid strings that
+ currently fail (e.g. ``2017-01-01T00:00+00:00:00``) are not
+ guaranteed to continue failing in future versions if they encode
+ a valid date.
+
+ .. versionadded:: 2.7.0
+ """
+ components, pos = self._parse_isodate(dt_str)
+
+ if len(dt_str) > pos:
+ if self._sep is None or dt_str[pos:pos + 1] == self._sep:
+ components += self._parse_isotime(dt_str[pos + 1:])
+ else:
+ raise ValueError('String contains unknown ISO components')
+
+ if len(components) > 3 and components[3] == 24:
+ components[3] = 0
+ return datetime(*components) + timedelta(days=1)
+
+ return datetime(*components)
+
+ @_takes_ascii
+ def parse_isodate(self, datestr):
+ """
+ Parse the date portion of an ISO string.
+
+ :param datestr:
+ The string portion of an ISO string, without a separator
+
+ :return:
+ Returns a :class:`datetime.date` object
+ """
+ components, pos = self._parse_isodate(datestr)
+ if pos < len(datestr):
+ raise ValueError('String contains unknown ISO ' +
+ 'components: {!r}'.format(datestr.decode('ascii')))
+ return date(*components)
+
+ @_takes_ascii
+ def parse_isotime(self, timestr):
+ """
+ Parse the time portion of an ISO string.
+
+ :param timestr:
+ The time portion of an ISO string, without a separator
+
+ :return:
+ Returns a :class:`datetime.time` object
+ """
+ components = self._parse_isotime(timestr)
+ if components[0] == 24:
+ components[0] = 0
+ return time(*components)
+
+ @_takes_ascii
+ def parse_tzstr(self, tzstr, zero_as_utc=True):
+ """
+ Parse a valid ISO time zone string.
+
+ See :func:`isoparser.isoparse` for details on supported formats.
+
+ :param tzstr:
+ A string representing an ISO time zone offset
+
+ :param zero_as_utc:
+ Whether to return :class:`dateutil.tz.tzutc` for zero-offset zones
+
+ :return:
+ Returns :class:`dateutil.tz.tzoffset` for offsets and
+ :class:`dateutil.tz.tzutc` for ``Z`` and (if ``zero_as_utc`` is
+ specified) offsets equivalent to UTC.
+ """
+ return self._parse_tzstr(tzstr, zero_as_utc=zero_as_utc)
+
+ # Constants
+ _DATE_SEP = b'-'
+ _TIME_SEP = b':'
+ _FRACTION_REGEX = re.compile(b'[\\.,]([0-9]+)')
+
+ def _parse_isodate(self, dt_str):
+ try:
+ return self._parse_isodate_common(dt_str)
+ except ValueError:
+ return self._parse_isodate_uncommon(dt_str)
+
+ def _parse_isodate_common(self, dt_str):
+ len_str = len(dt_str)
+ components = [1, 1, 1]
+
+ if len_str < 4:
+ raise ValueError('ISO string too short')
+
+ # Year
+ components[0] = int(dt_str[0:4])
+ pos = 4
+ if pos >= len_str:
+ return components, pos
+
+ has_sep = dt_str[pos:pos + 1] == self._DATE_SEP
+ if has_sep:
+ pos += 1
+
+ # Month
+ if len_str - pos < 2:
+ raise ValueError('Invalid common month')
+
+ components[1] = int(dt_str[pos:pos + 2])
+ pos += 2
+
+ if pos >= len_str:
+ if has_sep:
+ return components, pos
+ else:
+ raise ValueError('Invalid ISO format')
+
+ if has_sep:
+ if dt_str[pos:pos + 1] != self._DATE_SEP:
+ raise ValueError('Invalid separator in ISO string')
+ pos += 1
+
+ # Day
+ if len_str - pos < 2:
+ raise ValueError('Invalid common day')
+ components[2] = int(dt_str[pos:pos + 2])
+ return components, pos + 2
+
+ def _parse_isodate_uncommon(self, dt_str):
+ if len(dt_str) < 4:
+ raise ValueError('ISO string too short')
+
+ # All ISO formats start with the year
+ year = int(dt_str[0:4])
+
+ has_sep = dt_str[4:5] == self._DATE_SEP
+
+ pos = 4 + has_sep # Skip '-' if it's there
+ if dt_str[pos:pos + 1] == b'W':
+ # YYYY-?Www-?D?
+ pos += 1
+ weekno = int(dt_str[pos:pos + 2])
+ pos += 2
+
+ dayno = 1
+ if len(dt_str) > pos:
+ if (dt_str[pos:pos + 1] == self._DATE_SEP) != has_sep:
+ raise ValueError('Inconsistent use of dash separator')
+
+ pos += has_sep
+
+ dayno = int(dt_str[pos:pos + 1])
+ pos += 1
+
+ base_date = self._calculate_weekdate(year, weekno, dayno)
+ else:
+ # YYYYDDD or YYYY-DDD
+ if len(dt_str) - pos < 3:
+ raise ValueError('Invalid ordinal day')
+
+ ordinal_day = int(dt_str[pos:pos + 3])
+ pos += 3
+
+ if ordinal_day < 1 or ordinal_day > (365 + calendar.isleap(year)):
+ raise ValueError('Invalid ordinal day' +
+ ' {} for year {}'.format(ordinal_day, year))
+
+ base_date = date(year, 1, 1) + timedelta(days=ordinal_day - 1)
+
+ components = [base_date.year, base_date.month, base_date.day]
+ return components, pos
+
+ def _calculate_weekdate(self, year, week, day):
+ """
+ Calculate the day of corresponding to the ISO year-week-day calendar.
+
+ This function is effectively the inverse of
+ :func:`datetime.date.isocalendar`.
+
+ :param year:
+ The year in the ISO calendar
+
+ :param week:
+ The week in the ISO calendar - range is [1, 53]
+
+ :param day:
+ The day in the ISO calendar - range is [1 (MON), 7 (SUN)]
+
+ :return:
+ Returns a :class:`datetime.date`
+ """
+ if not 0 < week < 54:
+ raise ValueError('Invalid week: {}'.format(week))
+
+ if not 0 < day < 8: # Range is 1-7
+ raise ValueError('Invalid weekday: {}'.format(day))
+
+ # Get week 1 for the specific year:
+ jan_4 = date(year, 1, 4) # Week 1 always has January 4th in it
+ week_1 = jan_4 - timedelta(days=jan_4.isocalendar()[2] - 1)
+
+ # Now add the specific number of weeks and days to get what we want
+ week_offset = (week - 1) * 7 + (day - 1)
+ return week_1 + timedelta(days=week_offset)
+
+ def _parse_isotime(self, timestr):
+ len_str = len(timestr)
+ components = [0, 0, 0, 0, None]
+ pos = 0
+ comp = -1
+
+ if len_str < 2:
+ raise ValueError('ISO time too short')
+
+ has_sep = False
+
+ while pos < len_str and comp < 5:
+ comp += 1
+
+ if timestr[pos:pos + 1] in b'-+Zz':
+ # Detect time zone boundary
+ components[-1] = self._parse_tzstr(timestr[pos:])
+ pos = len_str
+ break
+
+ if comp == 1 and timestr[pos:pos+1] == self._TIME_SEP:
+ has_sep = True
+ pos += 1
+ elif comp == 2 and has_sep:
+ if timestr[pos:pos+1] != self._TIME_SEP:
+ raise ValueError('Inconsistent use of colon separator')
+ pos += 1
+
+ if comp < 3:
+ # Hour, minute, second
+ components[comp] = int(timestr[pos:pos + 2])
+ pos += 2
+
+ if comp == 3:
+ # Fraction of a second
+ frac = self._FRACTION_REGEX.match(timestr[pos:])
+ if not frac:
+ continue
+
+ us_str = frac.group(1)[:6] # Truncate to microseconds
+ components[comp] = int(us_str) * 10**(6 - len(us_str))
+ pos += len(frac.group())
+
+ if pos < len_str:
+ raise ValueError('Unused components in ISO string')
+
+ if components[0] == 24:
+ # Standard supports 00:00 and 24:00 as representations of midnight
+ if any(component != 0 for component in components[1:4]):
+ raise ValueError('Hour may only be 24 at 24:00:00.000')
+
+ return components
+
+ def _parse_tzstr(self, tzstr, zero_as_utc=True):
+ if tzstr == b'Z' or tzstr == b'z':
+ return tz.UTC
+
+ if len(tzstr) not in {3, 5, 6}:
+ raise ValueError('Time zone offset must be 1, 3, 5 or 6 characters')
+
+ if tzstr[0:1] == b'-':
+ mult = -1
+ elif tzstr[0:1] == b'+':
+ mult = 1
+ else:
+ raise ValueError('Time zone offset requires sign')
+
+ hours = int(tzstr[1:3])
+ if len(tzstr) == 3:
+ minutes = 0
+ else:
+ minutes = int(tzstr[(4 if tzstr[3:4] == self._TIME_SEP else 3):])
+
+ if zero_as_utc and hours == 0 and minutes == 0:
+ return tz.UTC
+ else:
+ if minutes > 59:
+ raise ValueError('Invalid minutes in time zone offset')
+
+ if hours > 23:
+ raise ValueError('Invalid hours in time zone offset')
+
+ return tz.tzoffset(None, mult * (hours * 60 + minutes) * 60)
+
+
+DEFAULT_ISOPARSER = isoparser()
+isoparse = DEFAULT_ISOPARSER.isoparse
diff --git a/venv/Lib/site-packages/dateutil/relativedelta.py b/venv/Lib/site-packages/dateutil/relativedelta.py
new file mode 100644
index 0000000000000000000000000000000000000000..cd323a549e0f182541ebcde2d2ea1adfbbd9701e
--- /dev/null
+++ b/venv/Lib/site-packages/dateutil/relativedelta.py
@@ -0,0 +1,599 @@
+# -*- coding: utf-8 -*-
+import datetime
+import calendar
+
+import operator
+from math import copysign
+
+from six import integer_types
+from warnings import warn
+
+from ._common import weekday
+
+MO, TU, WE, TH, FR, SA, SU = weekdays = tuple(weekday(x) for x in range(7))
+
+__all__ = ["relativedelta", "MO", "TU", "WE", "TH", "FR", "SA", "SU"]
+
+
+class relativedelta(object):
+ """
+ The relativedelta type is designed to be applied to an existing datetime and
+ can replace specific components of that datetime, or represents an interval
+ of time.
+
+ It is based on the specification of the excellent work done by M.-A. Lemburg
+ in his
+ `mx.DateTime `_ extension.
+ However, notice that this type does *NOT* implement the same algorithm as
+ his work. Do *NOT* expect it to behave like mx.DateTime's counterpart.
+
+ There are two different ways to build a relativedelta instance. The
+ first one is passing it two date/datetime classes::
+
+ relativedelta(datetime1, datetime2)
+
+ The second one is passing it any number of the following keyword arguments::
+
+ relativedelta(arg1=x,arg2=y,arg3=z...)
+
+ year, month, day, hour, minute, second, microsecond:
+ Absolute information (argument is singular); adding or subtracting a
+ relativedelta with absolute information does not perform an arithmetic
+ operation, but rather REPLACES the corresponding value in the
+ original datetime with the value(s) in relativedelta.
+
+ years, months, weeks, days, hours, minutes, seconds, microseconds:
+ Relative information, may be negative (argument is plural); adding
+ or subtracting a relativedelta with relative information performs
+ the corresponding arithmetic operation on the original datetime value
+ with the information in the relativedelta.
+
+ weekday:
+ One of the weekday instances (MO, TU, etc) available in the
+ relativedelta module. These instances may receive a parameter N,
+ specifying the Nth weekday, which could be positive or negative
+ (like MO(+1) or MO(-2)). Not specifying it is the same as specifying
+ +1. You can also use an integer, where 0=MO. This argument is always
+ relative e.g. if the calculated date is already Monday, using MO(1)
+ or MO(-1) won't change the day. To effectively make it absolute, use
+ it in combination with the day argument (e.g. day=1, MO(1) for first
+ Monday of the month).
+
+ leapdays:
+ Will add given days to the date found, if year is a leap
+ year, and the date found is post 28 of february.
+
+ yearday, nlyearday:
+ Set the yearday or the non-leap year day (jump leap days).
+ These are converted to day/month/leapdays information.
+
+ There are relative and absolute forms of the keyword
+ arguments. The plural is relative, and the singular is
+ absolute. For each argument in the order below, the absolute form
+ is applied first (by setting each attribute to that value) and
+ then the relative form (by adding the value to the attribute).
+
+ The order of attributes considered when this relativedelta is
+ added to a datetime is:
+
+ 1. Year
+ 2. Month
+ 3. Day
+ 4. Hours
+ 5. Minutes
+ 6. Seconds
+ 7. Microseconds
+
+ Finally, weekday is applied, using the rule described above.
+
+ For example
+
+ >>> from datetime import datetime
+ >>> from dateutil.relativedelta import relativedelta, MO
+ >>> dt = datetime(2018, 4, 9, 13, 37, 0)
+ >>> delta = relativedelta(hours=25, day=1, weekday=MO(1))
+ >>> dt + delta
+ datetime.datetime(2018, 4, 2, 14, 37)
+
+ First, the day is set to 1 (the first of the month), then 25 hours
+ are added, to get to the 2nd day and 14th hour, finally the
+ weekday is applied, but since the 2nd is already a Monday there is
+ no effect.
+
+ """
+
+ def __init__(self, dt1=None, dt2=None,
+ years=0, months=0, days=0, leapdays=0, weeks=0,
+ hours=0, minutes=0, seconds=0, microseconds=0,
+ year=None, month=None, day=None, weekday=None,
+ yearday=None, nlyearday=None,
+ hour=None, minute=None, second=None, microsecond=None):
+
+ if dt1 and dt2:
+ # datetime is a subclass of date. So both must be date
+ if not (isinstance(dt1, datetime.date) and
+ isinstance(dt2, datetime.date)):
+ raise TypeError("relativedelta only diffs datetime/date")
+
+ # We allow two dates, or two datetimes, so we coerce them to be
+ # of the same type
+ if (isinstance(dt1, datetime.datetime) !=
+ isinstance(dt2, datetime.datetime)):
+ if not isinstance(dt1, datetime.datetime):
+ dt1 = datetime.datetime.fromordinal(dt1.toordinal())
+ elif not isinstance(dt2, datetime.datetime):
+ dt2 = datetime.datetime.fromordinal(dt2.toordinal())
+
+ self.years = 0
+ self.months = 0
+ self.days = 0
+ self.leapdays = 0
+ self.hours = 0
+ self.minutes = 0
+ self.seconds = 0
+ self.microseconds = 0
+ self.year = None
+ self.month = None
+ self.day = None
+ self.weekday = None
+ self.hour = None
+ self.minute = None
+ self.second = None
+ self.microsecond = None
+ self._has_time = 0
+
+ # Get year / month delta between the two
+ months = (dt1.year - dt2.year) * 12 + (dt1.month - dt2.month)
+ self._set_months(months)
+
+ # Remove the year/month delta so the timedelta is just well-defined
+ # time units (seconds, days and microseconds)
+ dtm = self.__radd__(dt2)
+
+ # If we've overshot our target, make an adjustment
+ if dt1 < dt2:
+ compare = operator.gt
+ increment = 1
+ else:
+ compare = operator.lt
+ increment = -1
+
+ while compare(dt1, dtm):
+ months += increment
+ self._set_months(months)
+ dtm = self.__radd__(dt2)
+
+ # Get the timedelta between the "months-adjusted" date and dt1
+ delta = dt1 - dtm
+ self.seconds = delta.seconds + delta.days * 86400
+ self.microseconds = delta.microseconds
+ else:
+ # Check for non-integer values in integer-only quantities
+ if any(x is not None and x != int(x) for x in (years, months)):
+ raise ValueError("Non-integer years and months are "
+ "ambiguous and not currently supported.")
+
+ # Relative information
+ self.years = int(years)
+ self.months = int(months)
+ self.days = days + weeks * 7
+ self.leapdays = leapdays
+ self.hours = hours
+ self.minutes = minutes
+ self.seconds = seconds
+ self.microseconds = microseconds
+
+ # Absolute information
+ self.year = year
+ self.month = month
+ self.day = day
+ self.hour = hour
+ self.minute = minute
+ self.second = second
+ self.microsecond = microsecond
+
+ if any(x is not None and int(x) != x
+ for x in (year, month, day, hour,
+ minute, second, microsecond)):
+ # For now we'll deprecate floats - later it'll be an error.
+ warn("Non-integer value passed as absolute information. " +
+ "This is not a well-defined condition and will raise " +
+ "errors in future versions.", DeprecationWarning)
+
+ if isinstance(weekday, integer_types):
+ self.weekday = weekdays[weekday]
+ else:
+ self.weekday = weekday
+
+ yday = 0
+ if nlyearday:
+ yday = nlyearday
+ elif yearday:
+ yday = yearday
+ if yearday > 59:
+ self.leapdays = -1
+ if yday:
+ ydayidx = [31, 59, 90, 120, 151, 181, 212,
+ 243, 273, 304, 334, 366]
+ for idx, ydays in enumerate(ydayidx):
+ if yday <= ydays:
+ self.month = idx+1
+ if idx == 0:
+ self.day = yday
+ else:
+ self.day = yday-ydayidx[idx-1]
+ break
+ else:
+ raise ValueError("invalid year day (%d)" % yday)
+
+ self._fix()
+
+ def _fix(self):
+ if abs(self.microseconds) > 999999:
+ s = _sign(self.microseconds)
+ div, mod = divmod(self.microseconds * s, 1000000)
+ self.microseconds = mod * s
+ self.seconds += div * s
+ if abs(self.seconds) > 59:
+ s = _sign(self.seconds)
+ div, mod = divmod(self.seconds * s, 60)
+ self.seconds = mod * s
+ self.minutes += div * s
+ if abs(self.minutes) > 59:
+ s = _sign(self.minutes)
+ div, mod = divmod(self.minutes * s, 60)
+ self.minutes = mod * s
+ self.hours += div * s
+ if abs(self.hours) > 23:
+ s = _sign(self.hours)
+ div, mod = divmod(self.hours * s, 24)
+ self.hours = mod * s
+ self.days += div * s
+ if abs(self.months) > 11:
+ s = _sign(self.months)
+ div, mod = divmod(self.months * s, 12)
+ self.months = mod * s
+ self.years += div * s
+ if (self.hours or self.minutes or self.seconds or self.microseconds
+ or self.hour is not None or self.minute is not None or
+ self.second is not None or self.microsecond is not None):
+ self._has_time = 1
+ else:
+ self._has_time = 0
+
+ @property
+ def weeks(self):
+ return int(self.days / 7.0)
+
+ @weeks.setter
+ def weeks(self, value):
+ self.days = self.days - (self.weeks * 7) + value * 7
+
+ def _set_months(self, months):
+ self.months = months
+ if abs(self.months) > 11:
+ s = _sign(self.months)
+ div, mod = divmod(self.months * s, 12)
+ self.months = mod * s
+ self.years = div * s
+ else:
+ self.years = 0
+
+ def normalized(self):
+ """
+ Return a version of this object represented entirely using integer
+ values for the relative attributes.
+
+ >>> relativedelta(days=1.5, hours=2).normalized()
+ relativedelta(days=+1, hours=+14)
+
+ :return:
+ Returns a :class:`dateutil.relativedelta.relativedelta` object.
+ """
+ # Cascade remainders down (rounding each to roughly nearest microsecond)
+ days = int(self.days)
+
+ hours_f = round(self.hours + 24 * (self.days - days), 11)
+ hours = int(hours_f)
+
+ minutes_f = round(self.minutes + 60 * (hours_f - hours), 10)
+ minutes = int(minutes_f)
+
+ seconds_f = round(self.seconds + 60 * (minutes_f - minutes), 8)
+ seconds = int(seconds_f)
+
+ microseconds = round(self.microseconds + 1e6 * (seconds_f - seconds))
+
+ # Constructor carries overflow back up with call to _fix()
+ return self.__class__(years=self.years, months=self.months,
+ days=days, hours=hours, minutes=minutes,
+ seconds=seconds, microseconds=microseconds,
+ leapdays=self.leapdays, year=self.year,
+ month=self.month, day=self.day,
+ weekday=self.weekday, hour=self.hour,
+ minute=self.minute, second=self.second,
+ microsecond=self.microsecond)
+
+ def __add__(self, other):
+ if isinstance(other, relativedelta):
+ return self.__class__(years=other.years + self.years,
+ months=other.months + self.months,
+ days=other.days + self.days,
+ hours=other.hours + self.hours,
+ minutes=other.minutes + self.minutes,
+ seconds=other.seconds + self.seconds,
+ microseconds=(other.microseconds +
+ self.microseconds),
+ leapdays=other.leapdays or self.leapdays,
+ year=(other.year if other.year is not None
+ else self.year),
+ month=(other.month if other.month is not None
+ else self.month),
+ day=(other.day if other.day is not None
+ else self.day),
+ weekday=(other.weekday if other.weekday is not None
+ else self.weekday),
+ hour=(other.hour if other.hour is not None
+ else self.hour),
+ minute=(other.minute if other.minute is not None
+ else self.minute),
+ second=(other.second if other.second is not None
+ else self.second),
+ microsecond=(other.microsecond if other.microsecond
+ is not None else
+ self.microsecond))
+ if isinstance(other, datetime.timedelta):
+ return self.__class__(years=self.years,
+ months=self.months,
+ days=self.days + other.days,
+ hours=self.hours,
+ minutes=self.minutes,
+ seconds=self.seconds + other.seconds,
+ microseconds=self.microseconds + other.microseconds,
+ leapdays=self.leapdays,
+ year=self.year,
+ month=self.month,
+ day=self.day,
+ weekday=self.weekday,
+ hour=self.hour,
+ minute=self.minute,
+ second=self.second,
+ microsecond=self.microsecond)
+ if not isinstance(other, datetime.date):
+ return NotImplemented
+ elif self._has_time and not isinstance(other, datetime.datetime):
+ other = datetime.datetime.fromordinal(other.toordinal())
+ year = (self.year or other.year)+self.years
+ month = self.month or other.month
+ if self.months:
+ assert 1 <= abs(self.months) <= 12
+ month += self.months
+ if month > 12:
+ year += 1
+ month -= 12
+ elif month < 1:
+ year -= 1
+ month += 12
+ day = min(calendar.monthrange(year, month)[1],
+ self.day or other.day)
+ repl = {"year": year, "month": month, "day": day}
+ for attr in ["hour", "minute", "second", "microsecond"]:
+ value = getattr(self, attr)
+ if value is not None:
+ repl[attr] = value
+ days = self.days
+ if self.leapdays and month > 2 and calendar.isleap(year):
+ days += self.leapdays
+ ret = (other.replace(**repl)
+ + datetime.timedelta(days=days,
+ hours=self.hours,
+ minutes=self.minutes,
+ seconds=self.seconds,
+ microseconds=self.microseconds))
+ if self.weekday:
+ weekday, nth = self.weekday.weekday, self.weekday.n or 1
+ jumpdays = (abs(nth) - 1) * 7
+ if nth > 0:
+ jumpdays += (7 - ret.weekday() + weekday) % 7
+ else:
+ jumpdays += (ret.weekday() - weekday) % 7
+ jumpdays *= -1
+ ret += datetime.timedelta(days=jumpdays)
+ return ret
+
+ def __radd__(self, other):
+ return self.__add__(other)
+
+ def __rsub__(self, other):
+ return self.__neg__().__radd__(other)
+
+ def __sub__(self, other):
+ if not isinstance(other, relativedelta):
+ return NotImplemented # In case the other object defines __rsub__
+ return self.__class__(years=self.years - other.years,
+ months=self.months - other.months,
+ days=self.days - other.days,
+ hours=self.hours - other.hours,
+ minutes=self.minutes - other.minutes,
+ seconds=self.seconds - other.seconds,
+ microseconds=self.microseconds - other.microseconds,
+ leapdays=self.leapdays or other.leapdays,
+ year=(self.year if self.year is not None
+ else other.year),
+ month=(self.month if self.month is not None else
+ other.month),
+ day=(self.day if self.day is not None else
+ other.day),
+ weekday=(self.weekday if self.weekday is not None else
+ other.weekday),
+ hour=(self.hour if self.hour is not None else
+ other.hour),
+ minute=(self.minute if self.minute is not None else
+ other.minute),
+ second=(self.second if self.second is not None else
+ other.second),
+ microsecond=(self.microsecond if self.microsecond
+ is not None else
+ other.microsecond))
+
+ def __abs__(self):
+ return self.__class__(years=abs(self.years),
+ months=abs(self.months),
+ days=abs(self.days),
+ hours=abs(self.hours),
+ minutes=abs(self.minutes),
+ seconds=abs(self.seconds),
+ microseconds=abs(self.microseconds),
+ leapdays=self.leapdays,
+ year=self.year,
+ month=self.month,
+ day=self.day,
+ weekday=self.weekday,
+ hour=self.hour,
+ minute=self.minute,
+ second=self.second,
+ microsecond=self.microsecond)
+
+ def __neg__(self):
+ return self.__class__(years=-self.years,
+ months=-self.months,
+ days=-self.days,
+ hours=-self.hours,
+ minutes=-self.minutes,
+ seconds=-self.seconds,
+ microseconds=-self.microseconds,
+ leapdays=self.leapdays,
+ year=self.year,
+ month=self.month,
+ day=self.day,
+ weekday=self.weekday,
+ hour=self.hour,
+ minute=self.minute,
+ second=self.second,
+ microsecond=self.microsecond)
+
+ def __bool__(self):
+ return not (not self.years and
+ not self.months and
+ not self.days and
+ not self.hours and
+ not self.minutes and
+ not self.seconds and
+ not self.microseconds and
+ not self.leapdays and
+ self.year is None and
+ self.month is None and
+ self.day is None and
+ self.weekday is None and
+ self.hour is None and
+ self.minute is None and
+ self.second is None and
+ self.microsecond is None)
+ # Compatibility with Python 2.x
+ __nonzero__ = __bool__
+
+ def __mul__(self, other):
+ try:
+ f = float(other)
+ except TypeError:
+ return NotImplemented
+
+ return self.__class__(years=int(self.years * f),
+ months=int(self.months * f),
+ days=int(self.days * f),
+ hours=int(self.hours * f),
+ minutes=int(self.minutes * f),
+ seconds=int(self.seconds * f),
+ microseconds=int(self.microseconds * f),
+ leapdays=self.leapdays,
+ year=self.year,
+ month=self.month,
+ day=self.day,
+ weekday=self.weekday,
+ hour=self.hour,
+ minute=self.minute,
+ second=self.second,
+ microsecond=self.microsecond)
+
+ __rmul__ = __mul__
+
+ def __eq__(self, other):
+ if not isinstance(other, relativedelta):
+ return NotImplemented
+ if self.weekday or other.weekday:
+ if not self.weekday or not other.weekday:
+ return False
+ if self.weekday.weekday != other.weekday.weekday:
+ return False
+ n1, n2 = self.weekday.n, other.weekday.n
+ if n1 != n2 and not ((not n1 or n1 == 1) and (not n2 or n2 == 1)):
+ return False
+ return (self.years == other.years and
+ self.months == other.months and
+ self.days == other.days and
+ self.hours == other.hours and
+ self.minutes == other.minutes and
+ self.seconds == other.seconds and
+ self.microseconds == other.microseconds and
+ self.leapdays == other.leapdays and
+ self.year == other.year and
+ self.month == other.month and
+ self.day == other.day and
+ self.hour == other.hour and
+ self.minute == other.minute and
+ self.second == other.second and
+ self.microsecond == other.microsecond)
+
+ def __hash__(self):
+ return hash((
+ self.weekday,
+ self.years,
+ self.months,
+ self.days,
+ self.hours,
+ self.minutes,
+ self.seconds,
+ self.microseconds,
+ self.leapdays,
+ self.year,
+ self.month,
+ self.day,
+ self.hour,
+ self.minute,
+ self.second,
+ self.microsecond,
+ ))
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __div__(self, other):
+ try:
+ reciprocal = 1 / float(other)
+ except TypeError:
+ return NotImplemented
+
+ return self.__mul__(reciprocal)
+
+ __truediv__ = __div__
+
+ def __repr__(self):
+ l = []
+ for attr in ["years", "months", "days", "leapdays",
+ "hours", "minutes", "seconds", "microseconds"]:
+ value = getattr(self, attr)
+ if value:
+ l.append("{attr}={value:+g}".format(attr=attr, value=value))
+ for attr in ["year", "month", "day", "weekday",
+ "hour", "minute", "second", "microsecond"]:
+ value = getattr(self, attr)
+ if value is not None:
+ l.append("{attr}={value}".format(attr=attr, value=repr(value)))
+ return "{classname}({attrs})".format(classname=self.__class__.__name__,
+ attrs=", ".join(l))
+
+
+def _sign(x):
+ return int(copysign(1, x))
+
+# vim:ts=4:sw=4:et
diff --git a/venv/Lib/site-packages/dateutil/rrule.py b/venv/Lib/site-packages/dateutil/rrule.py
new file mode 100644
index 0000000000000000000000000000000000000000..571a0d2bc886a7ea4c06196b2f52e740c2ed6e9f
--- /dev/null
+++ b/venv/Lib/site-packages/dateutil/rrule.py
@@ -0,0 +1,1737 @@
+# -*- coding: utf-8 -*-
+"""
+The rrule module offers a small, complete, and very fast, implementation of
+the recurrence rules documented in the
+`iCalendar RFC `_,
+including support for caching of results.
+"""
+import calendar
+import datetime
+import heapq
+import itertools
+import re
+import sys
+from functools import wraps
+# For warning about deprecation of until and count
+from warnings import warn
+
+from six import advance_iterator, integer_types
+
+from six.moves import _thread, range
+
+from ._common import weekday as weekdaybase
+
+try:
+ from math import gcd
+except ImportError:
+ from fractions import gcd
+
+__all__ = ["rrule", "rruleset", "rrulestr",
+ "YEARLY", "MONTHLY", "WEEKLY", "DAILY",
+ "HOURLY", "MINUTELY", "SECONDLY",
+ "MO", "TU", "WE", "TH", "FR", "SA", "SU"]
+
+# Every mask is 7 days longer to handle cross-year weekly periods.
+M366MASK = tuple([1]*31+[2]*29+[3]*31+[4]*30+[5]*31+[6]*30 +
+ [7]*31+[8]*31+[9]*30+[10]*31+[11]*30+[12]*31+[1]*7)
+M365MASK = list(M366MASK)
+M29, M30, M31 = list(range(1, 30)), list(range(1, 31)), list(range(1, 32))
+MDAY366MASK = tuple(M31+M29+M31+M30+M31+M30+M31+M31+M30+M31+M30+M31+M31[:7])
+MDAY365MASK = list(MDAY366MASK)
+M29, M30, M31 = list(range(-29, 0)), list(range(-30, 0)), list(range(-31, 0))
+NMDAY366MASK = tuple(M31+M29+M31+M30+M31+M30+M31+M31+M30+M31+M30+M31+M31[:7])
+NMDAY365MASK = list(NMDAY366MASK)
+M366RANGE = (0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366)
+M365RANGE = (0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365)
+WDAYMASK = [0, 1, 2, 3, 4, 5, 6]*55
+del M29, M30, M31, M365MASK[59], MDAY365MASK[59], NMDAY365MASK[31]
+MDAY365MASK = tuple(MDAY365MASK)
+M365MASK = tuple(M365MASK)
+
+FREQNAMES = ['YEARLY', 'MONTHLY', 'WEEKLY', 'DAILY', 'HOURLY', 'MINUTELY', 'SECONDLY']
+
+(YEARLY,
+ MONTHLY,
+ WEEKLY,
+ DAILY,
+ HOURLY,
+ MINUTELY,
+ SECONDLY) = list(range(7))
+
+# Imported on demand.
+easter = None
+parser = None
+
+
+class weekday(weekdaybase):
+ """
+ This version of weekday does not allow n = 0.
+ """
+ def __init__(self, wkday, n=None):
+ if n == 0:
+ raise ValueError("Can't create weekday with n==0")
+
+ super(weekday, self).__init__(wkday, n)
+
+
+MO, TU, WE, TH, FR, SA, SU = weekdays = tuple(weekday(x) for x in range(7))
+
+
+def _invalidates_cache(f):
+ """
+ Decorator for rruleset methods which may invalidate the
+ cached length.
+ """
+ @wraps(f)
+ def inner_func(self, *args, **kwargs):
+ rv = f(self, *args, **kwargs)
+ self._invalidate_cache()
+ return rv
+
+ return inner_func
+
+
+class rrulebase(object):
+ def __init__(self, cache=False):
+ if cache:
+ self._cache = []
+ self._cache_lock = _thread.allocate_lock()
+ self._invalidate_cache()
+ else:
+ self._cache = None
+ self._cache_complete = False
+ self._len = None
+
+ def __iter__(self):
+ if self._cache_complete:
+ return iter(self._cache)
+ elif self._cache is None:
+ return self._iter()
+ else:
+ return self._iter_cached()
+
+ def _invalidate_cache(self):
+ if self._cache is not None:
+ self._cache = []
+ self._cache_complete = False
+ self._cache_gen = self._iter()
+
+ if self._cache_lock.locked():
+ self._cache_lock.release()
+
+ self._len = None
+
+ def _iter_cached(self):
+ i = 0
+ gen = self._cache_gen
+ cache = self._cache
+ acquire = self._cache_lock.acquire
+ release = self._cache_lock.release
+ while gen:
+ if i == len(cache):
+ acquire()
+ if self._cache_complete:
+ break
+ try:
+ for j in range(10):
+ cache.append(advance_iterator(gen))
+ except StopIteration:
+ self._cache_gen = gen = None
+ self._cache_complete = True
+ break
+ release()
+ yield cache[i]
+ i += 1
+ while i < self._len:
+ yield cache[i]
+ i += 1
+
+ def __getitem__(self, item):
+ if self._cache_complete:
+ return self._cache[item]
+ elif isinstance(item, slice):
+ if item.step and item.step < 0:
+ return list(iter(self))[item]
+ else:
+ return list(itertools.islice(self,
+ item.start or 0,
+ item.stop or sys.maxsize,
+ item.step or 1))
+ elif item >= 0:
+ gen = iter(self)
+ try:
+ for i in range(item+1):
+ res = advance_iterator(gen)
+ except StopIteration:
+ raise IndexError
+ return res
+ else:
+ return list(iter(self))[item]
+
+ def __contains__(self, item):
+ if self._cache_complete:
+ return item in self._cache
+ else:
+ for i in self:
+ if i == item:
+ return True
+ elif i > item:
+ return False
+ return False
+
+ # __len__() introduces a large performance penalty.
+ def count(self):
+ """ Returns the number of recurrences in this set. It will have go
+ through the whole recurrence, if this hasn't been done before. """
+ if self._len is None:
+ for x in self:
+ pass
+ return self._len
+
+ def before(self, dt, inc=False):
+ """ Returns the last recurrence before the given datetime instance. The
+ inc keyword defines what happens if dt is an occurrence. With
+ inc=True, if dt itself is an occurrence, it will be returned. """
+ if self._cache_complete:
+ gen = self._cache
+ else:
+ gen = self
+ last = None
+ if inc:
+ for i in gen:
+ if i > dt:
+ break
+ last = i
+ else:
+ for i in gen:
+ if i >= dt:
+ break
+ last = i
+ return last
+
+ def after(self, dt, inc=False):
+ """ Returns the first recurrence after the given datetime instance. The
+ inc keyword defines what happens if dt is an occurrence. With
+ inc=True, if dt itself is an occurrence, it will be returned. """
+ if self._cache_complete:
+ gen = self._cache
+ else:
+ gen = self
+ if inc:
+ for i in gen:
+ if i >= dt:
+ return i
+ else:
+ for i in gen:
+ if i > dt:
+ return i
+ return None
+
+ def xafter(self, dt, count=None, inc=False):
+ """
+ Generator which yields up to `count` recurrences after the given
+ datetime instance, equivalent to `after`.
+
+ :param dt:
+ The datetime at which to start generating recurrences.
+
+ :param count:
+ The maximum number of recurrences to generate. If `None` (default),
+ dates are generated until the recurrence rule is exhausted.
+
+ :param inc:
+ If `dt` is an instance of the rule and `inc` is `True`, it is
+ included in the output.
+
+ :yields: Yields a sequence of `datetime` objects.
+ """
+
+ if self._cache_complete:
+ gen = self._cache
+ else:
+ gen = self
+
+ # Select the comparison function
+ if inc:
+ comp = lambda dc, dtc: dc >= dtc
+ else:
+ comp = lambda dc, dtc: dc > dtc
+
+ # Generate dates
+ n = 0
+ for d in gen:
+ if comp(d, dt):
+ if count is not None:
+ n += 1
+ if n > count:
+ break
+
+ yield d
+
+ def between(self, after, before, inc=False, count=1):
+ """ Returns all the occurrences of the rrule between after and before.
+ The inc keyword defines what happens if after and/or before are
+ themselves occurrences. With inc=True, they will be included in the
+ list, if they are found in the recurrence set. """
+ if self._cache_complete:
+ gen = self._cache
+ else:
+ gen = self
+ started = False
+ l = []
+ if inc:
+ for i in gen:
+ if i > before:
+ break
+ elif not started:
+ if i >= after:
+ started = True
+ l.append(i)
+ else:
+ l.append(i)
+ else:
+ for i in gen:
+ if i >= before:
+ break
+ elif not started:
+ if i > after:
+ started = True
+ l.append(i)
+ else:
+ l.append(i)
+ return l
+
+
+class rrule(rrulebase):
+ """
+ That's the base of the rrule operation. It accepts all the keywords
+ defined in the RFC as its constructor parameters (except byday,
+ which was renamed to byweekday) and more. The constructor prototype is::
+
+ rrule(freq)
+
+ Where freq must be one of YEARLY, MONTHLY, WEEKLY, DAILY, HOURLY, MINUTELY,
+ or SECONDLY.
+
+ .. note::
+ Per RFC section 3.3.10, recurrence instances falling on invalid dates
+ and times are ignored rather than coerced:
+
+ Recurrence rules may generate recurrence instances with an invalid
+ date (e.g., February 30) or nonexistent local time (e.g., 1:30 AM
+ on a day where the local time is moved forward by an hour at 1:00
+ AM). Such recurrence instances MUST be ignored and MUST NOT be
+ counted as part of the recurrence set.
+
+ This can lead to possibly surprising behavior when, for example, the
+ start date occurs at the end of the month:
+
+ >>> from dateutil.rrule import rrule, MONTHLY
+ >>> from datetime import datetime
+ >>> start_date = datetime(2014, 12, 31)
+ >>> list(rrule(freq=MONTHLY, count=4, dtstart=start_date))
+ ... # doctest: +NORMALIZE_WHITESPACE
+ [datetime.datetime(2014, 12, 31, 0, 0),
+ datetime.datetime(2015, 1, 31, 0, 0),
+ datetime.datetime(2015, 3, 31, 0, 0),
+ datetime.datetime(2015, 5, 31, 0, 0)]
+
+ Additionally, it supports the following keyword arguments:
+
+ :param dtstart:
+ The recurrence start. Besides being the base for the recurrence,
+ missing parameters in the final recurrence instances will also be
+ extracted from this date. If not given, datetime.now() will be used
+ instead.
+ :param interval:
+ The interval between each freq iteration. For example, when using
+ YEARLY, an interval of 2 means once every two years, but with HOURLY,
+ it means once every two hours. The default interval is 1.
+ :param wkst:
+ The week start day. Must be one of the MO, TU, WE constants, or an
+ integer, specifying the first day of the week. This will affect
+ recurrences based on weekly periods. The default week start is got
+ from calendar.firstweekday(), and may be modified by
+ calendar.setfirstweekday().
+ :param count:
+ If given, this determines how many occurrences will be generated.
+
+ .. note::
+ As of version 2.5.0, the use of the keyword ``until`` in conjunction
+ with ``count`` is deprecated, to make sure ``dateutil`` is fully
+ compliant with `RFC-5545 Sec. 3.3.10 `_. Therefore, ``until`` and ``count``
+ **must not** occur in the same call to ``rrule``.
+ :param until:
+ If given, this must be a datetime instance specifying the upper-bound
+ limit of the recurrence. The last recurrence in the rule is the greatest
+ datetime that is less than or equal to the value specified in the
+ ``until`` parameter.
+
+ .. note::
+ As of version 2.5.0, the use of the keyword ``until`` in conjunction
+ with ``count`` is deprecated, to make sure ``dateutil`` is fully
+ compliant with `RFC-5545 Sec. 3.3.10 `_. Therefore, ``until`` and ``count``
+ **must not** occur in the same call to ``rrule``.
+ :param bysetpos:
+ If given, it must be either an integer, or a sequence of integers,
+ positive or negative. Each given integer will specify an occurrence
+ number, corresponding to the nth occurrence of the rule inside the
+ frequency period. For example, a bysetpos of -1 if combined with a
+ MONTHLY frequency, and a byweekday of (MO, TU, WE, TH, FR), will
+ result in the last work day of every month.
+ :param bymonth:
+ If given, it must be either an integer, or a sequence of integers,
+ meaning the months to apply the recurrence to.
+ :param bymonthday:
+ If given, it must be either an integer, or a sequence of integers,
+ meaning the month days to apply the recurrence to.
+ :param byyearday:
+ If given, it must be either an integer, or a sequence of integers,
+ meaning the year days to apply the recurrence to.
+ :param byeaster:
+ If given, it must be either an integer, or a sequence of integers,
+ positive or negative. Each integer will define an offset from the
+ Easter Sunday. Passing the offset 0 to byeaster will yield the Easter
+ Sunday itself. This is an extension to the RFC specification.
+ :param byweekno:
+ If given, it must be either an integer, or a sequence of integers,
+ meaning the week numbers to apply the recurrence to. Week numbers
+ have the meaning described in ISO8601, that is, the first week of
+ the year is that containing at least four days of the new year.
+ :param byweekday:
+ If given, it must be either an integer (0 == MO), a sequence of
+ integers, one of the weekday constants (MO, TU, etc), or a sequence
+ of these constants. When given, these variables will define the
+ weekdays where the recurrence will be applied. It's also possible to
+ use an argument n for the weekday instances, which will mean the nth
+ occurrence of this weekday in the period. For example, with MONTHLY,
+ or with YEARLY and BYMONTH, using FR(+1) in byweekday will specify the
+ first friday of the month where the recurrence happens. Notice that in
+ the RFC documentation, this is specified as BYDAY, but was renamed to
+ avoid the ambiguity of that keyword.
+ :param byhour:
+ If given, it must be either an integer, or a sequence of integers,
+ meaning the hours to apply the recurrence to.
+ :param byminute:
+ If given, it must be either an integer, or a sequence of integers,
+ meaning the minutes to apply the recurrence to.
+ :param bysecond:
+ If given, it must be either an integer, or a sequence of integers,
+ meaning the seconds to apply the recurrence to.
+ :param cache:
+ If given, it must be a boolean value specifying to enable or disable
+ caching of results. If you will use the same rrule instance multiple
+ times, enabling caching will improve the performance considerably.
+ """
+ def __init__(self, freq, dtstart=None,
+ interval=1, wkst=None, count=None, until=None, bysetpos=None,
+ bymonth=None, bymonthday=None, byyearday=None, byeaster=None,
+ byweekno=None, byweekday=None,
+ byhour=None, byminute=None, bysecond=None,
+ cache=False):
+ super(rrule, self).__init__(cache)
+ global easter
+ if not dtstart:
+ if until and until.tzinfo:
+ dtstart = datetime.datetime.now(tz=until.tzinfo).replace(microsecond=0)
+ else:
+ dtstart = datetime.datetime.now().replace(microsecond=0)
+ elif not isinstance(dtstart, datetime.datetime):
+ dtstart = datetime.datetime.fromordinal(dtstart.toordinal())
+ else:
+ dtstart = dtstart.replace(microsecond=0)
+ self._dtstart = dtstart
+ self._tzinfo = dtstart.tzinfo
+ self._freq = freq
+ self._interval = interval
+ self._count = count
+
+ # Cache the original byxxx rules, if they are provided, as the _byxxx
+ # attributes do not necessarily map to the inputs, and this can be
+ # a problem in generating the strings. Only store things if they've
+ # been supplied (the string retrieval will just use .get())
+ self._original_rule = {}
+
+ if until and not isinstance(until, datetime.datetime):
+ until = datetime.datetime.fromordinal(until.toordinal())
+ self._until = until
+
+ if self._dtstart and self._until:
+ if (self._dtstart.tzinfo is not None) != (self._until.tzinfo is not None):
+ # According to RFC5545 Section 3.3.10:
+ # https://tools.ietf.org/html/rfc5545#section-3.3.10
+ #
+ # > If the "DTSTART" property is specified as a date with UTC
+ # > time or a date with local time and time zone reference,
+ # > then the UNTIL rule part MUST be specified as a date with
+ # > UTC time.
+ raise ValueError(
+ 'RRULE UNTIL values must be specified in UTC when DTSTART '
+ 'is timezone-aware'
+ )
+
+ if count is not None and until:
+ warn("Using both 'count' and 'until' is inconsistent with RFC 5545"
+ " and has been deprecated in dateutil. Future versions will "
+ "raise an error.", DeprecationWarning)
+
+ if wkst is None:
+ self._wkst = calendar.firstweekday()
+ elif isinstance(wkst, integer_types):
+ self._wkst = wkst
+ else:
+ self._wkst = wkst.weekday
+
+ if bysetpos is None:
+ self._bysetpos = None
+ elif isinstance(bysetpos, integer_types):
+ if bysetpos == 0 or not (-366 <= bysetpos <= 366):
+ raise ValueError("bysetpos must be between 1 and 366, "
+ "or between -366 and -1")
+ self._bysetpos = (bysetpos,)
+ else:
+ self._bysetpos = tuple(bysetpos)
+ for pos in self._bysetpos:
+ if pos == 0 or not (-366 <= pos <= 366):
+ raise ValueError("bysetpos must be between 1 and 366, "
+ "or between -366 and -1")
+
+ if self._bysetpos:
+ self._original_rule['bysetpos'] = self._bysetpos
+
+ if (byweekno is None and byyearday is None and bymonthday is None and
+ byweekday is None and byeaster is None):
+ if freq == YEARLY:
+ if bymonth is None:
+ bymonth = dtstart.month
+ self._original_rule['bymonth'] = None
+ bymonthday = dtstart.day
+ self._original_rule['bymonthday'] = None
+ elif freq == MONTHLY:
+ bymonthday = dtstart.day
+ self._original_rule['bymonthday'] = None
+ elif freq == WEEKLY:
+ byweekday = dtstart.weekday()
+ self._original_rule['byweekday'] = None
+
+ # bymonth
+ if bymonth is None:
+ self._bymonth = None
+ else:
+ if isinstance(bymonth, integer_types):
+ bymonth = (bymonth,)
+
+ self._bymonth = tuple(sorted(set(bymonth)))
+
+ if 'bymonth' not in self._original_rule:
+ self._original_rule['bymonth'] = self._bymonth
+
+ # byyearday
+ if byyearday is None:
+ self._byyearday = None
+ else:
+ if isinstance(byyearday, integer_types):
+ byyearday = (byyearday,)
+
+ self._byyearday = tuple(sorted(set(byyearday)))
+ self._original_rule['byyearday'] = self._byyearday
+
+ # byeaster
+ if byeaster is not None:
+ if not easter:
+ from dateutil import easter
+ if isinstance(byeaster, integer_types):
+ self._byeaster = (byeaster,)
+ else:
+ self._byeaster = tuple(sorted(byeaster))
+
+ self._original_rule['byeaster'] = self._byeaster
+ else:
+ self._byeaster = None
+
+ # bymonthday
+ if bymonthday is None:
+ self._bymonthday = ()
+ self._bynmonthday = ()
+ else:
+ if isinstance(bymonthday, integer_types):
+ bymonthday = (bymonthday,)
+
+ bymonthday = set(bymonthday) # Ensure it's unique
+
+ self._bymonthday = tuple(sorted(x for x in bymonthday if x > 0))
+ self._bynmonthday = tuple(sorted(x for x in bymonthday if x < 0))
+
+ # Storing positive numbers first, then negative numbers
+ if 'bymonthday' not in self._original_rule:
+ self._original_rule['bymonthday'] = tuple(
+ itertools.chain(self._bymonthday, self._bynmonthday))
+
+ # byweekno
+ if byweekno is None:
+ self._byweekno = None
+ else:
+ if isinstance(byweekno, integer_types):
+ byweekno = (byweekno,)
+
+ self._byweekno = tuple(sorted(set(byweekno)))
+
+ self._original_rule['byweekno'] = self._byweekno
+
+ # byweekday / bynweekday
+ if byweekday is None:
+ self._byweekday = None
+ self._bynweekday = None
+ else:
+ # If it's one of the valid non-sequence types, convert to a
+ # single-element sequence before the iterator that builds the
+ # byweekday set.
+ if isinstance(byweekday, integer_types) or hasattr(byweekday, "n"):
+ byweekday = (byweekday,)
+
+ self._byweekday = set()
+ self._bynweekday = set()
+ for wday in byweekday:
+ if isinstance(wday, integer_types):
+ self._byweekday.add(wday)
+ elif not wday.n or freq > MONTHLY:
+ self._byweekday.add(wday.weekday)
+ else:
+ self._bynweekday.add((wday.weekday, wday.n))
+
+ if not self._byweekday:
+ self._byweekday = None
+ elif not self._bynweekday:
+ self._bynweekday = None
+
+ if self._byweekday is not None:
+ self._byweekday = tuple(sorted(self._byweekday))
+ orig_byweekday = [weekday(x) for x in self._byweekday]
+ else:
+ orig_byweekday = ()
+
+ if self._bynweekday is not None:
+ self._bynweekday = tuple(sorted(self._bynweekday))
+ orig_bynweekday = [weekday(*x) for x in self._bynweekday]
+ else:
+ orig_bynweekday = ()
+
+ if 'byweekday' not in self._original_rule:
+ self._original_rule['byweekday'] = tuple(itertools.chain(
+ orig_byweekday, orig_bynweekday))
+
+ # byhour
+ if byhour is None:
+ if freq < HOURLY:
+ self._byhour = {dtstart.hour}
+ else:
+ self._byhour = None
+ else:
+ if isinstance(byhour, integer_types):
+ byhour = (byhour,)
+
+ if freq == HOURLY:
+ self._byhour = self.__construct_byset(start=dtstart.hour,
+ byxxx=byhour,
+ base=24)
+ else:
+ self._byhour = set(byhour)
+
+ self._byhour = tuple(sorted(self._byhour))
+ self._original_rule['byhour'] = self._byhour
+
+ # byminute
+ if byminute is None:
+ if freq < MINUTELY:
+ self._byminute = {dtstart.minute}
+ else:
+ self._byminute = None
+ else:
+ if isinstance(byminute, integer_types):
+ byminute = (byminute,)
+
+ if freq == MINUTELY:
+ self._byminute = self.__construct_byset(start=dtstart.minute,
+ byxxx=byminute,
+ base=60)
+ else:
+ self._byminute = set(byminute)
+
+ self._byminute = tuple(sorted(self._byminute))
+ self._original_rule['byminute'] = self._byminute
+
+ # bysecond
+ if bysecond is None:
+ if freq < SECONDLY:
+ self._bysecond = ((dtstart.second,))
+ else:
+ self._bysecond = None
+ else:
+ if isinstance(bysecond, integer_types):
+ bysecond = (bysecond,)
+
+ self._bysecond = set(bysecond)
+
+ if freq == SECONDLY:
+ self._bysecond = self.__construct_byset(start=dtstart.second,
+ byxxx=bysecond,
+ base=60)
+ else:
+ self._bysecond = set(bysecond)
+
+ self._bysecond = tuple(sorted(self._bysecond))
+ self._original_rule['bysecond'] = self._bysecond
+
+ if self._freq >= HOURLY:
+ self._timeset = None
+ else:
+ self._timeset = []
+ for hour in self._byhour:
+ for minute in self._byminute:
+ for second in self._bysecond:
+ self._timeset.append(
+ datetime.time(hour, minute, second,
+ tzinfo=self._tzinfo))
+ self._timeset.sort()
+ self._timeset = tuple(self._timeset)
+
+ def __str__(self):
+ """
+ Output a string that would generate this RRULE if passed to rrulestr.
+ This is mostly compatible with RFC5545, except for the
+ dateutil-specific extension BYEASTER.
+ """
+
+ output = []
+ h, m, s = [None] * 3
+ if self._dtstart:
+ output.append(self._dtstart.strftime('DTSTART:%Y%m%dT%H%M%S'))
+ h, m, s = self._dtstart.timetuple()[3:6]
+
+ parts = ['FREQ=' + FREQNAMES[self._freq]]
+ if self._interval != 1:
+ parts.append('INTERVAL=' + str(self._interval))
+
+ if self._wkst:
+ parts.append('WKST=' + repr(weekday(self._wkst))[0:2])
+
+ if self._count is not None:
+ parts.append('COUNT=' + str(self._count))
+
+ if self._until:
+ parts.append(self._until.strftime('UNTIL=%Y%m%dT%H%M%S'))
+
+ if self._original_rule.get('byweekday') is not None:
+ # The str() method on weekday objects doesn't generate
+ # RFC5545-compliant strings, so we should modify that.
+ original_rule = dict(self._original_rule)
+ wday_strings = []
+ for wday in original_rule['byweekday']:
+ if wday.n:
+ wday_strings.append('{n:+d}{wday}'.format(
+ n=wday.n,
+ wday=repr(wday)[0:2]))
+ else:
+ wday_strings.append(repr(wday))
+
+ original_rule['byweekday'] = wday_strings
+ else:
+ original_rule = self._original_rule
+
+ partfmt = '{name}={vals}'
+ for name, key in [('BYSETPOS', 'bysetpos'),
+ ('BYMONTH', 'bymonth'),
+ ('BYMONTHDAY', 'bymonthday'),
+ ('BYYEARDAY', 'byyearday'),
+ ('BYWEEKNO', 'byweekno'),
+ ('BYDAY', 'byweekday'),
+ ('BYHOUR', 'byhour'),
+ ('BYMINUTE', 'byminute'),
+ ('BYSECOND', 'bysecond'),
+ ('BYEASTER', 'byeaster')]:
+ value = original_rule.get(key)
+ if value:
+ parts.append(partfmt.format(name=name, vals=(','.join(str(v)
+ for v in value))))
+
+ output.append('RRULE:' + ';'.join(parts))
+ return '\n'.join(output)
+
+ def replace(self, **kwargs):
+ """Return new rrule with same attributes except for those attributes given new
+ values by whichever keyword arguments are specified."""
+ new_kwargs = {"interval": self._interval,
+ "count": self._count,
+ "dtstart": self._dtstart,
+ "freq": self._freq,
+ "until": self._until,
+ "wkst": self._wkst,
+ "cache": False if self._cache is None else True }
+ new_kwargs.update(self._original_rule)
+ new_kwargs.update(kwargs)
+ return rrule(**new_kwargs)
+
+ def _iter(self):
+ year, month, day, hour, minute, second, weekday, yearday, _ = \
+ self._dtstart.timetuple()
+
+ # Some local variables to speed things up a bit
+ freq = self._freq
+ interval = self._interval
+ wkst = self._wkst
+ until = self._until
+ bymonth = self._bymonth
+ byweekno = self._byweekno
+ byyearday = self._byyearday
+ byweekday = self._byweekday
+ byeaster = self._byeaster
+ bymonthday = self._bymonthday
+ bynmonthday = self._bynmonthday
+ bysetpos = self._bysetpos
+ byhour = self._byhour
+ byminute = self._byminute
+ bysecond = self._bysecond
+
+ ii = _iterinfo(self)
+ ii.rebuild(year, month)
+
+ getdayset = {YEARLY: ii.ydayset,
+ MONTHLY: ii.mdayset,
+ WEEKLY: ii.wdayset,
+ DAILY: ii.ddayset,
+ HOURLY: ii.ddayset,
+ MINUTELY: ii.ddayset,
+ SECONDLY: ii.ddayset}[freq]
+
+ if freq < HOURLY:
+ timeset = self._timeset
+ else:
+ gettimeset = {HOURLY: ii.htimeset,
+ MINUTELY: ii.mtimeset,
+ SECONDLY: ii.stimeset}[freq]
+ if ((freq >= HOURLY and
+ self._byhour and hour not in self._byhour) or
+ (freq >= MINUTELY and
+ self._byminute and minute not in self._byminute) or
+ (freq >= SECONDLY and
+ self._bysecond and second not in self._bysecond)):
+ timeset = ()
+ else:
+ timeset = gettimeset(hour, minute, second)
+
+ total = 0
+ count = self._count
+ while True:
+ # Get dayset with the right frequency
+ dayset, start, end = getdayset(year, month, day)
+
+ # Do the "hard" work ;-)
+ filtered = False
+ for i in dayset[start:end]:
+ if ((bymonth and ii.mmask[i] not in bymonth) or
+ (byweekno and not ii.wnomask[i]) or
+ (byweekday and ii.wdaymask[i] not in byweekday) or
+ (ii.nwdaymask and not ii.nwdaymask[i]) or
+ (byeaster and not ii.eastermask[i]) or
+ ((bymonthday or bynmonthday) and
+ ii.mdaymask[i] not in bymonthday and
+ ii.nmdaymask[i] not in bynmonthday) or
+ (byyearday and
+ ((i < ii.yearlen and i+1 not in byyearday and
+ -ii.yearlen+i not in byyearday) or
+ (i >= ii.yearlen and i+1-ii.yearlen not in byyearday and
+ -ii.nextyearlen+i-ii.yearlen not in byyearday)))):
+ dayset[i] = None
+ filtered = True
+
+ # Output results
+ if bysetpos and timeset:
+ poslist = []
+ for pos in bysetpos:
+ if pos < 0:
+ daypos, timepos = divmod(pos, len(timeset))
+ else:
+ daypos, timepos = divmod(pos-1, len(timeset))
+ try:
+ i = [x for x in dayset[start:end]
+ if x is not None][daypos]
+ time = timeset[timepos]
+ except IndexError:
+ pass
+ else:
+ date = datetime.date.fromordinal(ii.yearordinal+i)
+ res = datetime.datetime.combine(date, time)
+ if res not in poslist:
+ poslist.append(res)
+ poslist.sort()
+ for res in poslist:
+ if until and res > until:
+ self._len = total
+ return
+ elif res >= self._dtstart:
+ if count is not None:
+ count -= 1
+ if count < 0:
+ self._len = total
+ return
+ total += 1
+ yield res
+ else:
+ for i in dayset[start:end]:
+ if i is not None:
+ date = datetime.date.fromordinal(ii.yearordinal + i)
+ for time in timeset:
+ res = datetime.datetime.combine(date, time)
+ if until and res > until:
+ self._len = total
+ return
+ elif res >= self._dtstart:
+ if count is not None:
+ count -= 1
+ if count < 0:
+ self._len = total
+ return
+
+ total += 1
+ yield res
+
+ # Handle frequency and interval
+ fixday = False
+ if freq == YEARLY:
+ year += interval
+ if year > datetime.MAXYEAR:
+ self._len = total
+ return
+ ii.rebuild(year, month)
+ elif freq == MONTHLY:
+ month += interval
+ if month > 12:
+ div, mod = divmod(month, 12)
+ month = mod
+ year += div
+ if month == 0:
+ month = 12
+ year -= 1
+ if year > datetime.MAXYEAR:
+ self._len = total
+ return
+ ii.rebuild(year, month)
+ elif freq == WEEKLY:
+ if wkst > weekday:
+ day += -(weekday+1+(6-wkst))+self._interval*7
+ else:
+ day += -(weekday-wkst)+self._interval*7
+ weekday = wkst
+ fixday = True
+ elif freq == DAILY:
+ day += interval
+ fixday = True
+ elif freq == HOURLY:
+ if filtered:
+ # Jump to one iteration before next day
+ hour += ((23-hour)//interval)*interval
+
+ if byhour:
+ ndays, hour = self.__mod_distance(value=hour,
+ byxxx=self._byhour,
+ base=24)
+ else:
+ ndays, hour = divmod(hour+interval, 24)
+
+ if ndays:
+ day += ndays
+ fixday = True
+
+ timeset = gettimeset(hour, minute, second)
+ elif freq == MINUTELY:
+ if filtered:
+ # Jump to one iteration before next day
+ minute += ((1439-(hour*60+minute))//interval)*interval
+
+ valid = False
+ rep_rate = (24*60)
+ for j in range(rep_rate // gcd(interval, rep_rate)):
+ if byminute:
+ nhours, minute = \
+ self.__mod_distance(value=minute,
+ byxxx=self._byminute,
+ base=60)
+ else:
+ nhours, minute = divmod(minute+interval, 60)
+
+ div, hour = divmod(hour+nhours, 24)
+ if div:
+ day += div
+ fixday = True
+ filtered = False
+
+ if not byhour or hour in byhour:
+ valid = True
+ break
+
+ if not valid:
+ raise ValueError('Invalid combination of interval and ' +
+ 'byhour resulting in empty rule.')
+
+ timeset = gettimeset(hour, minute, second)
+ elif freq == SECONDLY:
+ if filtered:
+ # Jump to one iteration before next day
+ second += (((86399 - (hour * 3600 + minute * 60 + second))
+ // interval) * interval)
+
+ rep_rate = (24 * 3600)
+ valid = False
+ for j in range(0, rep_rate // gcd(interval, rep_rate)):
+ if bysecond:
+ nminutes, second = \
+ self.__mod_distance(value=second,
+ byxxx=self._bysecond,
+ base=60)
+ else:
+ nminutes, second = divmod(second+interval, 60)
+
+ div, minute = divmod(minute+nminutes, 60)
+ if div:
+ hour += div
+ div, hour = divmod(hour, 24)
+ if div:
+ day += div
+ fixday = True
+
+ if ((not byhour or hour in byhour) and
+ (not byminute or minute in byminute) and
+ (not bysecond or second in bysecond)):
+ valid = True
+ break
+
+ if not valid:
+ raise ValueError('Invalid combination of interval, ' +
+ 'byhour and byminute resulting in empty' +
+ ' rule.')
+
+ timeset = gettimeset(hour, minute, second)
+
+ if fixday and day > 28:
+ daysinmonth = calendar.monthrange(year, month)[1]
+ if day > daysinmonth:
+ while day > daysinmonth:
+ day -= daysinmonth
+ month += 1
+ if month == 13:
+ month = 1
+ year += 1
+ if year > datetime.MAXYEAR:
+ self._len = total
+ return
+ daysinmonth = calendar.monthrange(year, month)[1]
+ ii.rebuild(year, month)
+
+ def __construct_byset(self, start, byxxx, base):
+ """
+ If a `BYXXX` sequence is passed to the constructor at the same level as
+ `FREQ` (e.g. `FREQ=HOURLY,BYHOUR={2,4,7},INTERVAL=3`), there are some
+ specifications which cannot be reached given some starting conditions.
+
+ This occurs whenever the interval is not coprime with the base of a
+ given unit and the difference between the starting position and the
+ ending position is not coprime with the greatest common denominator
+ between the interval and the base. For example, with a FREQ of hourly
+ starting at 17:00 and an interval of 4, the only valid values for
+ BYHOUR would be {21, 1, 5, 9, 13, 17}, because 4 and 24 are not
+ coprime.
+
+ :param start:
+ Specifies the starting position.
+ :param byxxx:
+ An iterable containing the list of allowed values.
+ :param base:
+ The largest allowable value for the specified frequency (e.g.
+ 24 hours, 60 minutes).
+
+ This does not preserve the type of the iterable, returning a set, since
+ the values should be unique and the order is irrelevant, this will
+ speed up later lookups.
+
+ In the event of an empty set, raises a :exception:`ValueError`, as this
+ results in an empty rrule.
+ """
+
+ cset = set()
+
+ # Support a single byxxx value.
+ if isinstance(byxxx, integer_types):
+ byxxx = (byxxx, )
+
+ for num in byxxx:
+ i_gcd = gcd(self._interval, base)
+ # Use divmod rather than % because we need to wrap negative nums.
+ if i_gcd == 1 or divmod(num - start, i_gcd)[1] == 0:
+ cset.add(num)
+
+ if len(cset) == 0:
+ raise ValueError("Invalid rrule byxxx generates an empty set.")
+
+ return cset
+
+ def __mod_distance(self, value, byxxx, base):
+ """
+ Calculates the next value in a sequence where the `FREQ` parameter is
+ specified along with a `BYXXX` parameter at the same "level"
+ (e.g. `HOURLY` specified with `BYHOUR`).
+
+ :param value:
+ The old value of the component.
+ :param byxxx:
+ The `BYXXX` set, which should have been generated by
+ `rrule._construct_byset`, or something else which checks that a
+ valid rule is present.
+ :param base:
+ The largest allowable value for the specified frequency (e.g.
+ 24 hours, 60 minutes).
+
+ If a valid value is not found after `base` iterations (the maximum
+ number before the sequence would start to repeat), this raises a
+ :exception:`ValueError`, as no valid values were found.
+
+ This returns a tuple of `divmod(n*interval, base)`, where `n` is the
+ smallest number of `interval` repetitions until the next specified
+ value in `byxxx` is found.
+ """
+ accumulator = 0
+ for ii in range(1, base + 1):
+ # Using divmod() over % to account for negative intervals
+ div, value = divmod(value + self._interval, base)
+ accumulator += div
+ if value in byxxx:
+ return (accumulator, value)
+
+
+class _iterinfo(object):
+ __slots__ = ["rrule", "lastyear", "lastmonth",
+ "yearlen", "nextyearlen", "yearordinal", "yearweekday",
+ "mmask", "mrange", "mdaymask", "nmdaymask",
+ "wdaymask", "wnomask", "nwdaymask", "eastermask"]
+
+ def __init__(self, rrule):
+ for attr in self.__slots__:
+ setattr(self, attr, None)
+ self.rrule = rrule
+
+ def rebuild(self, year, month):
+ # Every mask is 7 days longer to handle cross-year weekly periods.
+ rr = self.rrule
+ if year != self.lastyear:
+ self.yearlen = 365 + calendar.isleap(year)
+ self.nextyearlen = 365 + calendar.isleap(year + 1)
+ firstyday = datetime.date(year, 1, 1)
+ self.yearordinal = firstyday.toordinal()
+ self.yearweekday = firstyday.weekday()
+
+ wday = datetime.date(year, 1, 1).weekday()
+ if self.yearlen == 365:
+ self.mmask = M365MASK
+ self.mdaymask = MDAY365MASK
+ self.nmdaymask = NMDAY365MASK
+ self.wdaymask = WDAYMASK[wday:]
+ self.mrange = M365RANGE
+ else:
+ self.mmask = M366MASK
+ self.mdaymask = MDAY366MASK
+ self.nmdaymask = NMDAY366MASK
+ self.wdaymask = WDAYMASK[wday:]
+ self.mrange = M366RANGE
+
+ if not rr._byweekno:
+ self.wnomask = None
+ else:
+ self.wnomask = [0]*(self.yearlen+7)
+ # no1wkst = firstwkst = self.wdaymask.index(rr._wkst)
+ no1wkst = firstwkst = (7-self.yearweekday+rr._wkst) % 7
+ if no1wkst >= 4:
+ no1wkst = 0
+ # Number of days in the year, plus the days we got
+ # from last year.
+ wyearlen = self.yearlen+(self.yearweekday-rr._wkst) % 7
+ else:
+ # Number of days in the year, minus the days we
+ # left in last year.
+ wyearlen = self.yearlen-no1wkst
+ div, mod = divmod(wyearlen, 7)
+ numweeks = div+mod//4
+ for n in rr._byweekno:
+ if n < 0:
+ n += numweeks+1
+ if not (0 < n <= numweeks):
+ continue
+ if n > 1:
+ i = no1wkst+(n-1)*7
+ if no1wkst != firstwkst:
+ i -= 7-firstwkst
+ else:
+ i = no1wkst
+ for j in range(7):
+ self.wnomask[i] = 1
+ i += 1
+ if self.wdaymask[i] == rr._wkst:
+ break
+ if 1 in rr._byweekno:
+ # Check week number 1 of next year as well
+ # TODO: Check -numweeks for next year.
+ i = no1wkst+numweeks*7
+ if no1wkst != firstwkst:
+ i -= 7-firstwkst
+ if i < self.yearlen:
+ # If week starts in next year, we
+ # don't care about it.
+ for j in range(7):
+ self.wnomask[i] = 1
+ i += 1
+ if self.wdaymask[i] == rr._wkst:
+ break
+ if no1wkst:
+ # Check last week number of last year as
+ # well. If no1wkst is 0, either the year
+ # started on week start, or week number 1
+ # got days from last year, so there are no
+ # days from last year's last week number in
+ # this year.
+ if -1 not in rr._byweekno:
+ lyearweekday = datetime.date(year-1, 1, 1).weekday()
+ lno1wkst = (7-lyearweekday+rr._wkst) % 7
+ lyearlen = 365+calendar.isleap(year-1)
+ if lno1wkst >= 4:
+ lno1wkst = 0
+ lnumweeks = 52+(lyearlen +
+ (lyearweekday-rr._wkst) % 7) % 7//4
+ else:
+ lnumweeks = 52+(self.yearlen-no1wkst) % 7//4
+ else:
+ lnumweeks = -1
+ if lnumweeks in rr._byweekno:
+ for i in range(no1wkst):
+ self.wnomask[i] = 1
+
+ if (rr._bynweekday and (month != self.lastmonth or
+ year != self.lastyear)):
+ ranges = []
+ if rr._freq == YEARLY:
+ if rr._bymonth:
+ for month in rr._bymonth:
+ ranges.append(self.mrange[month-1:month+1])
+ else:
+ ranges = [(0, self.yearlen)]
+ elif rr._freq == MONTHLY:
+ ranges = [self.mrange[month-1:month+1]]
+ if ranges:
+ # Weekly frequency won't get here, so we may not
+ # care about cross-year weekly periods.
+ self.nwdaymask = [0]*self.yearlen
+ for first, last in ranges:
+ last -= 1
+ for wday, n in rr._bynweekday:
+ if n < 0:
+ i = last+(n+1)*7
+ i -= (self.wdaymask[i]-wday) % 7
+ else:
+ i = first+(n-1)*7
+ i += (7-self.wdaymask[i]+wday) % 7
+ if first <= i <= last:
+ self.nwdaymask[i] = 1
+
+ if rr._byeaster:
+ self.eastermask = [0]*(self.yearlen+7)
+ eyday = easter.easter(year).toordinal()-self.yearordinal
+ for offset in rr._byeaster:
+ self.eastermask[eyday+offset] = 1
+
+ self.lastyear = year
+ self.lastmonth = month
+
+ def ydayset(self, year, month, day):
+ return list(range(self.yearlen)), 0, self.yearlen
+
+ def mdayset(self, year, month, day):
+ dset = [None]*self.yearlen
+ start, end = self.mrange[month-1:month+1]
+ for i in range(start, end):
+ dset[i] = i
+ return dset, start, end
+
+ def wdayset(self, year, month, day):
+ # We need to handle cross-year weeks here.
+ dset = [None]*(self.yearlen+7)
+ i = datetime.date(year, month, day).toordinal()-self.yearordinal
+ start = i
+ for j in range(7):
+ dset[i] = i
+ i += 1
+ # if (not (0 <= i < self.yearlen) or
+ # self.wdaymask[i] == self.rrule._wkst):
+ # This will cross the year boundary, if necessary.
+ if self.wdaymask[i] == self.rrule._wkst:
+ break
+ return dset, start, i
+
+ def ddayset(self, year, month, day):
+ dset = [None] * self.yearlen
+ i = datetime.date(year, month, day).toordinal() - self.yearordinal
+ dset[i] = i
+ return dset, i, i + 1
+
+ def htimeset(self, hour, minute, second):
+ tset = []
+ rr = self.rrule
+ for minute in rr._byminute:
+ for second in rr._bysecond:
+ tset.append(datetime.time(hour, minute, second,
+ tzinfo=rr._tzinfo))
+ tset.sort()
+ return tset
+
+ def mtimeset(self, hour, minute, second):
+ tset = []
+ rr = self.rrule
+ for second in rr._bysecond:
+ tset.append(datetime.time(hour, minute, second, tzinfo=rr._tzinfo))
+ tset.sort()
+ return tset
+
+ def stimeset(self, hour, minute, second):
+ return (datetime.time(hour, minute, second,
+ tzinfo=self.rrule._tzinfo),)
+
+
+class rruleset(rrulebase):
+ """ The rruleset type allows more complex recurrence setups, mixing
+ multiple rules, dates, exclusion rules, and exclusion dates. The type
+ constructor takes the following keyword arguments:
+
+ :param cache: If True, caching of results will be enabled, improving
+ performance of multiple queries considerably. """
+
+ class _genitem(object):
+ def __init__(self, genlist, gen):
+ try:
+ self.dt = advance_iterator(gen)
+ genlist.append(self)
+ except StopIteration:
+ pass
+ self.genlist = genlist
+ self.gen = gen
+
+ def __next__(self):
+ try:
+ self.dt = advance_iterator(self.gen)
+ except StopIteration:
+ if self.genlist[0] is self:
+ heapq.heappop(self.genlist)
+ else:
+ self.genlist.remove(self)
+ heapq.heapify(self.genlist)
+
+ next = __next__
+
+ def __lt__(self, other):
+ return self.dt < other.dt
+
+ def __gt__(self, other):
+ return self.dt > other.dt
+
+ def __eq__(self, other):
+ return self.dt == other.dt
+
+ def __ne__(self, other):
+ return self.dt != other.dt
+
+ def __init__(self, cache=False):
+ super(rruleset, self).__init__(cache)
+ self._rrule = []
+ self._rdate = []
+ self._exrule = []
+ self._exdate = []
+
+ @_invalidates_cache
+ def rrule(self, rrule):
+ """ Include the given :py:class:`rrule` instance in the recurrence set
+ generation. """
+ self._rrule.append(rrule)
+
+ @_invalidates_cache
+ def rdate(self, rdate):
+ """ Include the given :py:class:`datetime` instance in the recurrence
+ set generation. """
+ self._rdate.append(rdate)
+
+ @_invalidates_cache
+ def exrule(self, exrule):
+ """ Include the given rrule instance in the recurrence set exclusion
+ list. Dates which are part of the given recurrence rules will not
+ be generated, even if some inclusive rrule or rdate matches them.
+ """
+ self._exrule.append(exrule)
+
+ @_invalidates_cache
+ def exdate(self, exdate):
+ """ Include the given datetime instance in the recurrence set
+ exclusion list. Dates included that way will not be generated,
+ even if some inclusive rrule or rdate matches them. """
+ self._exdate.append(exdate)
+
+ def _iter(self):
+ rlist = []
+ self._rdate.sort()
+ self._genitem(rlist, iter(self._rdate))
+ for gen in [iter(x) for x in self._rrule]:
+ self._genitem(rlist, gen)
+ exlist = []
+ self._exdate.sort()
+ self._genitem(exlist, iter(self._exdate))
+ for gen in [iter(x) for x in self._exrule]:
+ self._genitem(exlist, gen)
+ lastdt = None
+ total = 0
+ heapq.heapify(rlist)
+ heapq.heapify(exlist)
+ while rlist:
+ ritem = rlist[0]
+ if not lastdt or lastdt != ritem.dt:
+ while exlist and exlist[0] < ritem:
+ exitem = exlist[0]
+ advance_iterator(exitem)
+ if exlist and exlist[0] is exitem:
+ heapq.heapreplace(exlist, exitem)
+ if not exlist or ritem != exlist[0]:
+ total += 1
+ yield ritem.dt
+ lastdt = ritem.dt
+ advance_iterator(ritem)
+ if rlist and rlist[0] is ritem:
+ heapq.heapreplace(rlist, ritem)
+ self._len = total
+
+
+
+
+class _rrulestr(object):
+ """ Parses a string representation of a recurrence rule or set of
+ recurrence rules.
+
+ :param s:
+ Required, a string defining one or more recurrence rules.
+
+ :param dtstart:
+ If given, used as the default recurrence start if not specified in the
+ rule string.
+
+ :param cache:
+ If set ``True`` caching of results will be enabled, improving
+ performance of multiple queries considerably.
+
+ :param unfold:
+ If set ``True`` indicates that a rule string is split over more
+ than one line and should be joined before processing.
+
+ :param forceset:
+ If set ``True`` forces a :class:`dateutil.rrule.rruleset` to
+ be returned.
+
+ :param compatible:
+ If set ``True`` forces ``unfold`` and ``forceset`` to be ``True``.
+
+ :param ignoretz:
+ If set ``True``, time zones in parsed strings are ignored and a naive
+ :class:`datetime.datetime` object is returned.
+
+ :param tzids:
+ If given, a callable or mapping used to retrieve a
+ :class:`datetime.tzinfo` from a string representation.
+ Defaults to :func:`dateutil.tz.gettz`.
+
+ :param tzinfos:
+ Additional time zone names / aliases which may be present in a string
+ representation. See :func:`dateutil.parser.parse` for more
+ information.
+
+ :return:
+ Returns a :class:`dateutil.rrule.rruleset` or
+ :class:`dateutil.rrule.rrule`
+ """
+
+ _freq_map = {"YEARLY": YEARLY,
+ "MONTHLY": MONTHLY,
+ "WEEKLY": WEEKLY,
+ "DAILY": DAILY,
+ "HOURLY": HOURLY,
+ "MINUTELY": MINUTELY,
+ "SECONDLY": SECONDLY}
+
+ _weekday_map = {"MO": 0, "TU": 1, "WE": 2, "TH": 3,
+ "FR": 4, "SA": 5, "SU": 6}
+
+ def _handle_int(self, rrkwargs, name, value, **kwargs):
+ rrkwargs[name.lower()] = int(value)
+
+ def _handle_int_list(self, rrkwargs, name, value, **kwargs):
+ rrkwargs[name.lower()] = [int(x) for x in value.split(',')]
+
+ _handle_INTERVAL = _handle_int
+ _handle_COUNT = _handle_int
+ _handle_BYSETPOS = _handle_int_list
+ _handle_BYMONTH = _handle_int_list
+ _handle_BYMONTHDAY = _handle_int_list
+ _handle_BYYEARDAY = _handle_int_list
+ _handle_BYEASTER = _handle_int_list
+ _handle_BYWEEKNO = _handle_int_list
+ _handle_BYHOUR = _handle_int_list
+ _handle_BYMINUTE = _handle_int_list
+ _handle_BYSECOND = _handle_int_list
+
+ def _handle_FREQ(self, rrkwargs, name, value, **kwargs):
+ rrkwargs["freq"] = self._freq_map[value]
+
+ def _handle_UNTIL(self, rrkwargs, name, value, **kwargs):
+ global parser
+ if not parser:
+ from dateutil import parser
+ try:
+ rrkwargs["until"] = parser.parse(value,
+ ignoretz=kwargs.get("ignoretz"),
+ tzinfos=kwargs.get("tzinfos"))
+ except ValueError:
+ raise ValueError("invalid until date")
+
+ def _handle_WKST(self, rrkwargs, name, value, **kwargs):
+ rrkwargs["wkst"] = self._weekday_map[value]
+
+ def _handle_BYWEEKDAY(self, rrkwargs, name, value, **kwargs):
+ """
+ Two ways to specify this: +1MO or MO(+1)
+ """
+ l = []
+ for wday in value.split(','):
+ if '(' in wday:
+ # If it's of the form TH(+1), etc.
+ splt = wday.split('(')
+ w = splt[0]
+ n = int(splt[1][:-1])
+ elif len(wday):
+ # If it's of the form +1MO
+ for i in range(len(wday)):
+ if wday[i] not in '+-0123456789':
+ break
+ n = wday[:i] or None
+ w = wday[i:]
+ if n:
+ n = int(n)
+ else:
+ raise ValueError("Invalid (empty) BYDAY specification.")
+
+ l.append(weekdays[self._weekday_map[w]](n))
+ rrkwargs["byweekday"] = l
+
+ _handle_BYDAY = _handle_BYWEEKDAY
+
+ def _parse_rfc_rrule(self, line,
+ dtstart=None,
+ cache=False,
+ ignoretz=False,
+ tzinfos=None):
+ if line.find(':') != -1:
+ name, value = line.split(':')
+ if name != "RRULE":
+ raise ValueError("unknown parameter name")
+ else:
+ value = line
+ rrkwargs = {}
+ for pair in value.split(';'):
+ name, value = pair.split('=')
+ name = name.upper()
+ value = value.upper()
+ try:
+ getattr(self, "_handle_"+name)(rrkwargs, name, value,
+ ignoretz=ignoretz,
+ tzinfos=tzinfos)
+ except AttributeError:
+ raise ValueError("unknown parameter '%s'" % name)
+ except (KeyError, ValueError):
+ raise ValueError("invalid '%s': %s" % (name, value))
+ return rrule(dtstart=dtstart, cache=cache, **rrkwargs)
+
+ def _parse_date_value(self, date_value, parms, rule_tzids,
+ ignoretz, tzids, tzinfos):
+ global parser
+ if not parser:
+ from dateutil import parser
+
+ datevals = []
+ value_found = False
+ TZID = None
+
+ for parm in parms:
+ if parm.startswith("TZID="):
+ try:
+ tzkey = rule_tzids[parm.split('TZID=')[-1]]
+ except KeyError:
+ continue
+ if tzids is None:
+ from . import tz
+ tzlookup = tz.gettz
+ elif callable(tzids):
+ tzlookup = tzids
+ else:
+ tzlookup = getattr(tzids, 'get', None)
+ if tzlookup is None:
+ msg = ('tzids must be a callable, mapping, or None, '
+ 'not %s' % tzids)
+ raise ValueError(msg)
+
+ TZID = tzlookup(tzkey)
+ continue
+
+ # RFC 5445 3.8.2.4: The VALUE parameter is optional, but may be found
+ # only once.
+ if parm not in {"VALUE=DATE-TIME", "VALUE=DATE"}:
+ raise ValueError("unsupported parm: " + parm)
+ else:
+ if value_found:
+ msg = ("Duplicate value parameter found in: " + parm)
+ raise ValueError(msg)
+ value_found = True
+
+ for datestr in date_value.split(','):
+ date = parser.parse(datestr, ignoretz=ignoretz, tzinfos=tzinfos)
+ if TZID is not None:
+ if date.tzinfo is None:
+ date = date.replace(tzinfo=TZID)
+ else:
+ raise ValueError('DTSTART/EXDATE specifies multiple timezone')
+ datevals.append(date)
+
+ return datevals
+
+ def _parse_rfc(self, s,
+ dtstart=None,
+ cache=False,
+ unfold=False,
+ forceset=False,
+ compatible=False,
+ ignoretz=False,
+ tzids=None,
+ tzinfos=None):
+ global parser
+ if compatible:
+ forceset = True
+ unfold = True
+
+ TZID_NAMES = dict(map(
+ lambda x: (x.upper(), x),
+ re.findall('TZID=(?P[^:]+):', s)
+ ))
+ s = s.upper()
+ if not s.strip():
+ raise ValueError("empty string")
+ if unfold:
+ lines = s.splitlines()
+ i = 0
+ while i < len(lines):
+ line = lines[i].rstrip()
+ if not line:
+ del lines[i]
+ elif i > 0 and line[0] == " ":
+ lines[i-1] += line[1:]
+ del lines[i]
+ else:
+ i += 1
+ else:
+ lines = s.split()
+ if (not forceset and len(lines) == 1 and (s.find(':') == -1 or
+ s.startswith('RRULE:'))):
+ return self._parse_rfc_rrule(lines[0], cache=cache,
+ dtstart=dtstart, ignoretz=ignoretz,
+ tzinfos=tzinfos)
+ else:
+ rrulevals = []
+ rdatevals = []
+ exrulevals = []
+ exdatevals = []
+ for line in lines:
+ if not line:
+ continue
+ if line.find(':') == -1:
+ name = "RRULE"
+ value = line
+ else:
+ name, value = line.split(':', 1)
+ parms = name.split(';')
+ if not parms:
+ raise ValueError("empty property name")
+ name = parms[0]
+ parms = parms[1:]
+ if name == "RRULE":
+ for parm in parms:
+ raise ValueError("unsupported RRULE parm: "+parm)
+ rrulevals.append(value)
+ elif name == "RDATE":
+ for parm in parms:
+ if parm != "VALUE=DATE-TIME":
+ raise ValueError("unsupported RDATE parm: "+parm)
+ rdatevals.append(value)
+ elif name == "EXRULE":
+ for parm in parms:
+ raise ValueError("unsupported EXRULE parm: "+parm)
+ exrulevals.append(value)
+ elif name == "EXDATE":
+ exdatevals.extend(
+ self._parse_date_value(value, parms,
+ TZID_NAMES, ignoretz,
+ tzids, tzinfos)
+ )
+ elif name == "DTSTART":
+ dtvals = self._parse_date_value(value, parms, TZID_NAMES,
+ ignoretz, tzids, tzinfos)
+ if len(dtvals) != 1:
+ raise ValueError("Multiple DTSTART values specified:" +
+ value)
+ dtstart = dtvals[0]
+ else:
+ raise ValueError("unsupported property: "+name)
+ if (forceset or len(rrulevals) > 1 or rdatevals
+ or exrulevals or exdatevals):
+ if not parser and (rdatevals or exdatevals):
+ from dateutil import parser
+ rset = rruleset(cache=cache)
+ for value in rrulevals:
+ rset.rrule(self._parse_rfc_rrule(value, dtstart=dtstart,
+ ignoretz=ignoretz,
+ tzinfos=tzinfos))
+ for value in rdatevals:
+ for datestr in value.split(','):
+ rset.rdate(parser.parse(datestr,
+ ignoretz=ignoretz,
+ tzinfos=tzinfos))
+ for value in exrulevals:
+ rset.exrule(self._parse_rfc_rrule(value, dtstart=dtstart,
+ ignoretz=ignoretz,
+ tzinfos=tzinfos))
+ for value in exdatevals:
+ rset.exdate(value)
+ if compatible and dtstart:
+ rset.rdate(dtstart)
+ return rset
+ else:
+ return self._parse_rfc_rrule(rrulevals[0],
+ dtstart=dtstart,
+ cache=cache,
+ ignoretz=ignoretz,
+ tzinfos=tzinfos)
+
+ def __call__(self, s, **kwargs):
+ return self._parse_rfc(s, **kwargs)
+
+
+rrulestr = _rrulestr()
+
+# vim:ts=4:sw=4:et
diff --git a/venv/Lib/site-packages/dateutil/tz/__init__.py b/venv/Lib/site-packages/dateutil/tz/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..af1352c47292f4eebc5cae8da45641b5544558e3
--- /dev/null
+++ b/venv/Lib/site-packages/dateutil/tz/__init__.py
@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+from .tz import *
+from .tz import __doc__
+
+__all__ = ["tzutc", "tzoffset", "tzlocal", "tzfile", "tzrange",
+ "tzstr", "tzical", "tzwin", "tzwinlocal", "gettz",
+ "enfold", "datetime_ambiguous", "datetime_exists",
+ "resolve_imaginary", "UTC", "DeprecatedTzFormatWarning"]
+
+
+class DeprecatedTzFormatWarning(Warning):
+ """Warning raised when time zones are parsed from deprecated formats."""
diff --git a/venv/Lib/site-packages/dateutil/tz/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/dateutil/tz/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c246d7c4573dee2b5108e5621959fc2901c2940d
Binary files /dev/null and b/venv/Lib/site-packages/dateutil/tz/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dateutil/tz/__pycache__/_common.cpython-311.pyc b/venv/Lib/site-packages/dateutil/tz/__pycache__/_common.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..428ca742600408b510f3271ca9546831b6973325
Binary files /dev/null and b/venv/Lib/site-packages/dateutil/tz/__pycache__/_common.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dateutil/tz/__pycache__/_factories.cpython-311.pyc b/venv/Lib/site-packages/dateutil/tz/__pycache__/_factories.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..42dcbef7b64ef2a42177eafc4e29dea2bf90a908
Binary files /dev/null and b/venv/Lib/site-packages/dateutil/tz/__pycache__/_factories.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dateutil/tz/__pycache__/tz.cpython-311.pyc b/venv/Lib/site-packages/dateutil/tz/__pycache__/tz.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b3c3684147769cb4dbccf64db8207ac60ebd5c38
Binary files /dev/null and b/venv/Lib/site-packages/dateutil/tz/__pycache__/tz.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dateutil/tz/__pycache__/win.cpython-311.pyc b/venv/Lib/site-packages/dateutil/tz/__pycache__/win.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e5e0fcf52ca74ea1182ba96e330e93a4c719cf64
Binary files /dev/null and b/venv/Lib/site-packages/dateutil/tz/__pycache__/win.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dateutil/tz/_common.py b/venv/Lib/site-packages/dateutil/tz/_common.py
new file mode 100644
index 0000000000000000000000000000000000000000..e6ac11831522b266114d5b68ee1da298e3aeb14a
--- /dev/null
+++ b/venv/Lib/site-packages/dateutil/tz/_common.py
@@ -0,0 +1,419 @@
+from six import PY2
+
+from functools import wraps
+
+from datetime import datetime, timedelta, tzinfo
+
+
+ZERO = timedelta(0)
+
+__all__ = ['tzname_in_python2', 'enfold']
+
+
+def tzname_in_python2(namefunc):
+ """Change unicode output into bytestrings in Python 2
+
+ tzname() API changed in Python 3. It used to return bytes, but was changed
+ to unicode strings
+ """
+ if PY2:
+ @wraps(namefunc)
+ def adjust_encoding(*args, **kwargs):
+ name = namefunc(*args, **kwargs)
+ if name is not None:
+ name = name.encode()
+
+ return name
+
+ return adjust_encoding
+ else:
+ return namefunc
+
+
+# The following is adapted from Alexander Belopolsky's tz library
+# https://github.com/abalkin/tz
+if hasattr(datetime, 'fold'):
+ # This is the pre-python 3.6 fold situation
+ def enfold(dt, fold=1):
+ """
+ Provides a unified interface for assigning the ``fold`` attribute to
+ datetimes both before and after the implementation of PEP-495.
+
+ :param fold:
+ The value for the ``fold`` attribute in the returned datetime. This
+ should be either 0 or 1.
+
+ :return:
+ Returns an object for which ``getattr(dt, 'fold', 0)`` returns
+ ``fold`` for all versions of Python. In versions prior to
+ Python 3.6, this is a ``_DatetimeWithFold`` object, which is a
+ subclass of :py:class:`datetime.datetime` with the ``fold``
+ attribute added, if ``fold`` is 1.
+
+ .. versionadded:: 2.6.0
+ """
+ return dt.replace(fold=fold)
+
+else:
+ class _DatetimeWithFold(datetime):
+ """
+ This is a class designed to provide a PEP 495-compliant interface for
+ Python versions before 3.6. It is used only for dates in a fold, so
+ the ``fold`` attribute is fixed at ``1``.
+
+ .. versionadded:: 2.6.0
+ """
+ __slots__ = ()
+
+ def replace(self, *args, **kwargs):
+ """
+ Return a datetime with the same attributes, except for those
+ attributes given new values by whichever keyword arguments are
+ specified. Note that tzinfo=None can be specified to create a naive
+ datetime from an aware datetime with no conversion of date and time
+ data.
+
+ This is reimplemented in ``_DatetimeWithFold`` because pypy3 will
+ return a ``datetime.datetime`` even if ``fold`` is unchanged.
+ """
+ argnames = (
+ 'year', 'month', 'day', 'hour', 'minute', 'second',
+ 'microsecond', 'tzinfo'
+ )
+
+ for arg, argname in zip(args, argnames):
+ if argname in kwargs:
+ raise TypeError('Duplicate argument: {}'.format(argname))
+
+ kwargs[argname] = arg
+
+ for argname in argnames:
+ if argname not in kwargs:
+ kwargs[argname] = getattr(self, argname)
+
+ dt_class = self.__class__ if kwargs.get('fold', 1) else datetime
+
+ return dt_class(**kwargs)
+
+ @property
+ def fold(self):
+ return 1
+
+ def enfold(dt, fold=1):
+ """
+ Provides a unified interface for assigning the ``fold`` attribute to
+ datetimes both before and after the implementation of PEP-495.
+
+ :param fold:
+ The value for the ``fold`` attribute in the returned datetime. This
+ should be either 0 or 1.
+
+ :return:
+ Returns an object for which ``getattr(dt, 'fold', 0)`` returns
+ ``fold`` for all versions of Python. In versions prior to
+ Python 3.6, this is a ``_DatetimeWithFold`` object, which is a
+ subclass of :py:class:`datetime.datetime` with the ``fold``
+ attribute added, if ``fold`` is 1.
+
+ .. versionadded:: 2.6.0
+ """
+ if getattr(dt, 'fold', 0) == fold:
+ return dt
+
+ args = dt.timetuple()[:6]
+ args += (dt.microsecond, dt.tzinfo)
+
+ if fold:
+ return _DatetimeWithFold(*args)
+ else:
+ return datetime(*args)
+
+
+def _validate_fromutc_inputs(f):
+ """
+ The CPython version of ``fromutc`` checks that the input is a ``datetime``
+ object and that ``self`` is attached as its ``tzinfo``.
+ """
+ @wraps(f)
+ def fromutc(self, dt):
+ if not isinstance(dt, datetime):
+ raise TypeError("fromutc() requires a datetime argument")
+ if dt.tzinfo is not self:
+ raise ValueError("dt.tzinfo is not self")
+
+ return f(self, dt)
+
+ return fromutc
+
+
+class _tzinfo(tzinfo):
+ """
+ Base class for all ``dateutil`` ``tzinfo`` objects.
+ """
+
+ def is_ambiguous(self, dt):
+ """
+ Whether or not the "wall time" of a given datetime is ambiguous in this
+ zone.
+
+ :param dt:
+ A :py:class:`datetime.datetime`, naive or time zone aware.
+
+
+ :return:
+ Returns ``True`` if ambiguous, ``False`` otherwise.
+
+ .. versionadded:: 2.6.0
+ """
+
+ dt = dt.replace(tzinfo=self)
+
+ wall_0 = enfold(dt, fold=0)
+ wall_1 = enfold(dt, fold=1)
+
+ same_offset = wall_0.utcoffset() == wall_1.utcoffset()
+ same_dt = wall_0.replace(tzinfo=None) == wall_1.replace(tzinfo=None)
+
+ return same_dt and not same_offset
+
+ def _fold_status(self, dt_utc, dt_wall):
+ """
+ Determine the fold status of a "wall" datetime, given a representation
+ of the same datetime as a (naive) UTC datetime. This is calculated based
+ on the assumption that ``dt.utcoffset() - dt.dst()`` is constant for all
+ datetimes, and that this offset is the actual number of hours separating
+ ``dt_utc`` and ``dt_wall``.
+
+ :param dt_utc:
+ Representation of the datetime as UTC
+
+ :param dt_wall:
+ Representation of the datetime as "wall time". This parameter must
+ either have a `fold` attribute or have a fold-naive
+ :class:`datetime.tzinfo` attached, otherwise the calculation may
+ fail.
+ """
+ if self.is_ambiguous(dt_wall):
+ delta_wall = dt_wall - dt_utc
+ _fold = int(delta_wall == (dt_utc.utcoffset() - dt_utc.dst()))
+ else:
+ _fold = 0
+
+ return _fold
+
+ def _fold(self, dt):
+ return getattr(dt, 'fold', 0)
+
+ def _fromutc(self, dt):
+ """
+ Given a timezone-aware datetime in a given timezone, calculates a
+ timezone-aware datetime in a new timezone.
+
+ Since this is the one time that we *know* we have an unambiguous
+ datetime object, we take this opportunity to determine whether the
+ datetime is ambiguous and in a "fold" state (e.g. if it's the first
+ occurrence, chronologically, of the ambiguous datetime).
+
+ :param dt:
+ A timezone-aware :class:`datetime.datetime` object.
+ """
+
+ # Re-implement the algorithm from Python's datetime.py
+ dtoff = dt.utcoffset()
+ if dtoff is None:
+ raise ValueError("fromutc() requires a non-None utcoffset() "
+ "result")
+
+ # The original datetime.py code assumes that `dst()` defaults to
+ # zero during ambiguous times. PEP 495 inverts this presumption, so
+ # for pre-PEP 495 versions of python, we need to tweak the algorithm.
+ dtdst = dt.dst()
+ if dtdst is None:
+ raise ValueError("fromutc() requires a non-None dst() result")
+ delta = dtoff - dtdst
+
+ dt += delta
+ # Set fold=1 so we can default to being in the fold for
+ # ambiguous dates.
+ dtdst = enfold(dt, fold=1).dst()
+ if dtdst is None:
+ raise ValueError("fromutc(): dt.dst gave inconsistent "
+ "results; cannot convert")
+ return dt + dtdst
+
+ @_validate_fromutc_inputs
+ def fromutc(self, dt):
+ """
+ Given a timezone-aware datetime in a given timezone, calculates a
+ timezone-aware datetime in a new timezone.
+
+ Since this is the one time that we *know* we have an unambiguous
+ datetime object, we take this opportunity to determine whether the
+ datetime is ambiguous and in a "fold" state (e.g. if it's the first
+ occurrence, chronologically, of the ambiguous datetime).
+
+ :param dt:
+ A timezone-aware :class:`datetime.datetime` object.
+ """
+ dt_wall = self._fromutc(dt)
+
+ # Calculate the fold status given the two datetimes.
+ _fold = self._fold_status(dt, dt_wall)
+
+ # Set the default fold value for ambiguous dates
+ return enfold(dt_wall, fold=_fold)
+
+
+class tzrangebase(_tzinfo):
+ """
+ This is an abstract base class for time zones represented by an annual
+ transition into and out of DST. Child classes should implement the following
+ methods:
+
+ * ``__init__(self, *args, **kwargs)``
+ * ``transitions(self, year)`` - this is expected to return a tuple of
+ datetimes representing the DST on and off transitions in standard
+ time.
+
+ A fully initialized ``tzrangebase`` subclass should also provide the
+ following attributes:
+ * ``hasdst``: Boolean whether or not the zone uses DST.
+ * ``_dst_offset`` / ``_std_offset``: :class:`datetime.timedelta` objects
+ representing the respective UTC offsets.
+ * ``_dst_abbr`` / ``_std_abbr``: Strings representing the timezone short
+ abbreviations in DST and STD, respectively.
+ * ``_hasdst``: Whether or not the zone has DST.
+
+ .. versionadded:: 2.6.0
+ """
+ def __init__(self):
+ raise NotImplementedError('tzrangebase is an abstract base class')
+
+ def utcoffset(self, dt):
+ isdst = self._isdst(dt)
+
+ if isdst is None:
+ return None
+ elif isdst:
+ return self._dst_offset
+ else:
+ return self._std_offset
+
+ def dst(self, dt):
+ isdst = self._isdst(dt)
+
+ if isdst is None:
+ return None
+ elif isdst:
+ return self._dst_base_offset
+ else:
+ return ZERO
+
+ @tzname_in_python2
+ def tzname(self, dt):
+ if self._isdst(dt):
+ return self._dst_abbr
+ else:
+ return self._std_abbr
+
+ def fromutc(self, dt):
+ """ Given a datetime in UTC, return local time """
+ if not isinstance(dt, datetime):
+ raise TypeError("fromutc() requires a datetime argument")
+
+ if dt.tzinfo is not self:
+ raise ValueError("dt.tzinfo is not self")
+
+ # Get transitions - if there are none, fixed offset
+ transitions = self.transitions(dt.year)
+ if transitions is None:
+ return dt + self.utcoffset(dt)
+
+ # Get the transition times in UTC
+ dston, dstoff = transitions
+
+ dston -= self._std_offset
+ dstoff -= self._std_offset
+
+ utc_transitions = (dston, dstoff)
+ dt_utc = dt.replace(tzinfo=None)
+
+ isdst = self._naive_isdst(dt_utc, utc_transitions)
+
+ if isdst:
+ dt_wall = dt + self._dst_offset
+ else:
+ dt_wall = dt + self._std_offset
+
+ _fold = int(not isdst and self.is_ambiguous(dt_wall))
+
+ return enfold(dt_wall, fold=_fold)
+
+ def is_ambiguous(self, dt):
+ """
+ Whether or not the "wall time" of a given datetime is ambiguous in this
+ zone.
+
+ :param dt:
+ A :py:class:`datetime.datetime`, naive or time zone aware.
+
+
+ :return:
+ Returns ``True`` if ambiguous, ``False`` otherwise.
+
+ .. versionadded:: 2.6.0
+ """
+ if not self.hasdst:
+ return False
+
+ start, end = self.transitions(dt.year)
+
+ dt = dt.replace(tzinfo=None)
+ return (end <= dt < end + self._dst_base_offset)
+
+ def _isdst(self, dt):
+ if not self.hasdst:
+ return False
+ elif dt is None:
+ return None
+
+ transitions = self.transitions(dt.year)
+
+ if transitions is None:
+ return False
+
+ dt = dt.replace(tzinfo=None)
+
+ isdst = self._naive_isdst(dt, transitions)
+
+ # Handle ambiguous dates
+ if not isdst and self.is_ambiguous(dt):
+ return not self._fold(dt)
+ else:
+ return isdst
+
+ def _naive_isdst(self, dt, transitions):
+ dston, dstoff = transitions
+
+ dt = dt.replace(tzinfo=None)
+
+ if dston < dstoff:
+ isdst = dston <= dt < dstoff
+ else:
+ isdst = not dstoff <= dt < dston
+
+ return isdst
+
+ @property
+ def _dst_base_offset(self):
+ return self._dst_offset - self._std_offset
+
+ __hash__ = None
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ def __repr__(self):
+ return "%s(...)" % self.__class__.__name__
+
+ __reduce__ = object.__reduce__
diff --git a/venv/Lib/site-packages/dateutil/tz/_factories.py b/venv/Lib/site-packages/dateutil/tz/_factories.py
new file mode 100644
index 0000000000000000000000000000000000000000..f8a65891a023ebf9eb0c24d391ba67541b7133f1
--- /dev/null
+++ b/venv/Lib/site-packages/dateutil/tz/_factories.py
@@ -0,0 +1,80 @@
+from datetime import timedelta
+import weakref
+from collections import OrderedDict
+
+from six.moves import _thread
+
+
+class _TzSingleton(type):
+ def __init__(cls, *args, **kwargs):
+ cls.__instance = None
+ super(_TzSingleton, cls).__init__(*args, **kwargs)
+
+ def __call__(cls):
+ if cls.__instance is None:
+ cls.__instance = super(_TzSingleton, cls).__call__()
+ return cls.__instance
+
+
+class _TzFactory(type):
+ def instance(cls, *args, **kwargs):
+ """Alternate constructor that returns a fresh instance"""
+ return type.__call__(cls, *args, **kwargs)
+
+
+class _TzOffsetFactory(_TzFactory):
+ def __init__(cls, *args, **kwargs):
+ cls.__instances = weakref.WeakValueDictionary()
+ cls.__strong_cache = OrderedDict()
+ cls.__strong_cache_size = 8
+
+ cls._cache_lock = _thread.allocate_lock()
+
+ def __call__(cls, name, offset):
+ if isinstance(offset, timedelta):
+ key = (name, offset.total_seconds())
+ else:
+ key = (name, offset)
+
+ instance = cls.__instances.get(key, None)
+ if instance is None:
+ instance = cls.__instances.setdefault(key,
+ cls.instance(name, offset))
+
+ # This lock may not be necessary in Python 3. See GH issue #901
+ with cls._cache_lock:
+ cls.__strong_cache[key] = cls.__strong_cache.pop(key, instance)
+
+ # Remove an item if the strong cache is overpopulated
+ if len(cls.__strong_cache) > cls.__strong_cache_size:
+ cls.__strong_cache.popitem(last=False)
+
+ return instance
+
+
+class _TzStrFactory(_TzFactory):
+ def __init__(cls, *args, **kwargs):
+ cls.__instances = weakref.WeakValueDictionary()
+ cls.__strong_cache = OrderedDict()
+ cls.__strong_cache_size = 8
+
+ cls.__cache_lock = _thread.allocate_lock()
+
+ def __call__(cls, s, posix_offset=False):
+ key = (s, posix_offset)
+ instance = cls.__instances.get(key, None)
+
+ if instance is None:
+ instance = cls.__instances.setdefault(key,
+ cls.instance(s, posix_offset))
+
+ # This lock may not be necessary in Python 3. See GH issue #901
+ with cls.__cache_lock:
+ cls.__strong_cache[key] = cls.__strong_cache.pop(key, instance)
+
+ # Remove an item if the strong cache is overpopulated
+ if len(cls.__strong_cache) > cls.__strong_cache_size:
+ cls.__strong_cache.popitem(last=False)
+
+ return instance
+
diff --git a/venv/Lib/site-packages/dateutil/tz/tz.py b/venv/Lib/site-packages/dateutil/tz/tz.py
new file mode 100644
index 0000000000000000000000000000000000000000..617591446bd92eb1cc7b7d67fa3f17435e691cdd
--- /dev/null
+++ b/venv/Lib/site-packages/dateutil/tz/tz.py
@@ -0,0 +1,1849 @@
+# -*- coding: utf-8 -*-
+"""
+This module offers timezone implementations subclassing the abstract
+:py:class:`datetime.tzinfo` type. There are classes to handle tzfile format
+files (usually are in :file:`/etc/localtime`, :file:`/usr/share/zoneinfo`,
+etc), TZ environment string (in all known formats), given ranges (with help
+from relative deltas), local machine timezone, fixed offset timezone, and UTC
+timezone.
+"""
+import datetime
+import struct
+import time
+import sys
+import os
+import bisect
+import weakref
+from collections import OrderedDict
+
+import six
+from six import string_types
+from six.moves import _thread
+from ._common import tzname_in_python2, _tzinfo
+from ._common import tzrangebase, enfold
+from ._common import _validate_fromutc_inputs
+
+from ._factories import _TzSingleton, _TzOffsetFactory
+from ._factories import _TzStrFactory
+try:
+ from .win import tzwin, tzwinlocal
+except ImportError:
+ tzwin = tzwinlocal = None
+
+# For warning about rounding tzinfo
+from warnings import warn
+
+ZERO = datetime.timedelta(0)
+EPOCH = datetime.datetime(1970, 1, 1, 0, 0)
+EPOCHORDINAL = EPOCH.toordinal()
+
+
+@six.add_metaclass(_TzSingleton)
+class tzutc(datetime.tzinfo):
+ """
+ This is a tzinfo object that represents the UTC time zone.
+
+ **Examples:**
+
+ .. doctest::
+
+ >>> from datetime import *
+ >>> from dateutil.tz import *
+
+ >>> datetime.now()
+ datetime.datetime(2003, 9, 27, 9, 40, 1, 521290)
+
+ >>> datetime.now(tzutc())
+ datetime.datetime(2003, 9, 27, 12, 40, 12, 156379, tzinfo=tzutc())
+
+ >>> datetime.now(tzutc()).tzname()
+ 'UTC'
+
+ .. versionchanged:: 2.7.0
+ ``tzutc()`` is now a singleton, so the result of ``tzutc()`` will
+ always return the same object.
+
+ .. doctest::
+
+ >>> from dateutil.tz import tzutc, UTC
+ >>> tzutc() is tzutc()
+ True
+ >>> tzutc() is UTC
+ True
+ """
+ def utcoffset(self, dt):
+ return ZERO
+
+ def dst(self, dt):
+ return ZERO
+
+ @tzname_in_python2
+ def tzname(self, dt):
+ return "UTC"
+
+ def is_ambiguous(self, dt):
+ """
+ Whether or not the "wall time" of a given datetime is ambiguous in this
+ zone.
+
+ :param dt:
+ A :py:class:`datetime.datetime`, naive or time zone aware.
+
+
+ :return:
+ Returns ``True`` if ambiguous, ``False`` otherwise.
+
+ .. versionadded:: 2.6.0
+ """
+ return False
+
+ @_validate_fromutc_inputs
+ def fromutc(self, dt):
+ """
+ Fast track version of fromutc() returns the original ``dt`` object for
+ any valid :py:class:`datetime.datetime` object.
+ """
+ return dt
+
+ def __eq__(self, other):
+ if not isinstance(other, (tzutc, tzoffset)):
+ return NotImplemented
+
+ return (isinstance(other, tzutc) or
+ (isinstance(other, tzoffset) and other._offset == ZERO))
+
+ __hash__ = None
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ def __repr__(self):
+ return "%s()" % self.__class__.__name__
+
+ __reduce__ = object.__reduce__
+
+
+#: Convenience constant providing a :class:`tzutc()` instance
+#:
+#: .. versionadded:: 2.7.0
+UTC = tzutc()
+
+
+@six.add_metaclass(_TzOffsetFactory)
+class tzoffset(datetime.tzinfo):
+ """
+ A simple class for representing a fixed offset from UTC.
+
+ :param name:
+ The timezone name, to be returned when ``tzname()`` is called.
+ :param offset:
+ The time zone offset in seconds, or (since version 2.6.0, represented
+ as a :py:class:`datetime.timedelta` object).
+ """
+ def __init__(self, name, offset):
+ self._name = name
+
+ try:
+ # Allow a timedelta
+ offset = offset.total_seconds()
+ except (TypeError, AttributeError):
+ pass
+
+ self._offset = datetime.timedelta(seconds=_get_supported_offset(offset))
+
+ def utcoffset(self, dt):
+ return self._offset
+
+ def dst(self, dt):
+ return ZERO
+
+ @tzname_in_python2
+ def tzname(self, dt):
+ return self._name
+
+ @_validate_fromutc_inputs
+ def fromutc(self, dt):
+ return dt + self._offset
+
+ def is_ambiguous(self, dt):
+ """
+ Whether or not the "wall time" of a given datetime is ambiguous in this
+ zone.
+
+ :param dt:
+ A :py:class:`datetime.datetime`, naive or time zone aware.
+ :return:
+ Returns ``True`` if ambiguous, ``False`` otherwise.
+
+ .. versionadded:: 2.6.0
+ """
+ return False
+
+ def __eq__(self, other):
+ if not isinstance(other, tzoffset):
+ return NotImplemented
+
+ return self._offset == other._offset
+
+ __hash__ = None
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ def __repr__(self):
+ return "%s(%s, %s)" % (self.__class__.__name__,
+ repr(self._name),
+ int(self._offset.total_seconds()))
+
+ __reduce__ = object.__reduce__
+
+
+class tzlocal(_tzinfo):
+ """
+ A :class:`tzinfo` subclass built around the ``time`` timezone functions.
+ """
+ def __init__(self):
+ super(tzlocal, self).__init__()
+
+ self._std_offset = datetime.timedelta(seconds=-time.timezone)
+ if time.daylight:
+ self._dst_offset = datetime.timedelta(seconds=-time.altzone)
+ else:
+ self._dst_offset = self._std_offset
+
+ self._dst_saved = self._dst_offset - self._std_offset
+ self._hasdst = bool(self._dst_saved)
+ self._tznames = tuple(time.tzname)
+
+ def utcoffset(self, dt):
+ if dt is None and self._hasdst:
+ return None
+
+ if self._isdst(dt):
+ return self._dst_offset
+ else:
+ return self._std_offset
+
+ def dst(self, dt):
+ if dt is None and self._hasdst:
+ return None
+
+ if self._isdst(dt):
+ return self._dst_offset - self._std_offset
+ else:
+ return ZERO
+
+ @tzname_in_python2
+ def tzname(self, dt):
+ return self._tznames[self._isdst(dt)]
+
+ def is_ambiguous(self, dt):
+ """
+ Whether or not the "wall time" of a given datetime is ambiguous in this
+ zone.
+
+ :param dt:
+ A :py:class:`datetime.datetime`, naive or time zone aware.
+
+
+ :return:
+ Returns ``True`` if ambiguous, ``False`` otherwise.
+
+ .. versionadded:: 2.6.0
+ """
+ naive_dst = self._naive_is_dst(dt)
+ return (not naive_dst and
+ (naive_dst != self._naive_is_dst(dt - self._dst_saved)))
+
+ def _naive_is_dst(self, dt):
+ timestamp = _datetime_to_timestamp(dt)
+ return time.localtime(timestamp + time.timezone).tm_isdst
+
+ def _isdst(self, dt, fold_naive=True):
+ # We can't use mktime here. It is unstable when deciding if
+ # the hour near to a change is DST or not.
+ #
+ # timestamp = time.mktime((dt.year, dt.month, dt.day, dt.hour,
+ # dt.minute, dt.second, dt.weekday(), 0, -1))
+ # return time.localtime(timestamp).tm_isdst
+ #
+ # The code above yields the following result:
+ #
+ # >>> import tz, datetime
+ # >>> t = tz.tzlocal()
+ # >>> datetime.datetime(2003,2,15,23,tzinfo=t).tzname()
+ # 'BRDT'
+ # >>> datetime.datetime(2003,2,16,0,tzinfo=t).tzname()
+ # 'BRST'
+ # >>> datetime.datetime(2003,2,15,23,tzinfo=t).tzname()
+ # 'BRST'
+ # >>> datetime.datetime(2003,2,15,22,tzinfo=t).tzname()
+ # 'BRDT'
+ # >>> datetime.datetime(2003,2,15,23,tzinfo=t).tzname()
+ # 'BRDT'
+ #
+ # Here is a more stable implementation:
+ #
+ if not self._hasdst:
+ return False
+
+ # Check for ambiguous times:
+ dstval = self._naive_is_dst(dt)
+ fold = getattr(dt, 'fold', None)
+
+ if self.is_ambiguous(dt):
+ if fold is not None:
+ return not self._fold(dt)
+ else:
+ return True
+
+ return dstval
+
+ def __eq__(self, other):
+ if isinstance(other, tzlocal):
+ return (self._std_offset == other._std_offset and
+ self._dst_offset == other._dst_offset)
+ elif isinstance(other, tzutc):
+ return (not self._hasdst and
+ self._tznames[0] in {'UTC', 'GMT'} and
+ self._std_offset == ZERO)
+ elif isinstance(other, tzoffset):
+ return (not self._hasdst and
+ self._tznames[0] == other._name and
+ self._std_offset == other._offset)
+ else:
+ return NotImplemented
+
+ __hash__ = None
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ def __repr__(self):
+ return "%s()" % self.__class__.__name__
+
+ __reduce__ = object.__reduce__
+
+
+class _ttinfo(object):
+ __slots__ = ["offset", "delta", "isdst", "abbr",
+ "isstd", "isgmt", "dstoffset"]
+
+ def __init__(self):
+ for attr in self.__slots__:
+ setattr(self, attr, None)
+
+ def __repr__(self):
+ l = []
+ for attr in self.__slots__:
+ value = getattr(self, attr)
+ if value is not None:
+ l.append("%s=%s" % (attr, repr(value)))
+ return "%s(%s)" % (self.__class__.__name__, ", ".join(l))
+
+ def __eq__(self, other):
+ if not isinstance(other, _ttinfo):
+ return NotImplemented
+
+ return (self.offset == other.offset and
+ self.delta == other.delta and
+ self.isdst == other.isdst and
+ self.abbr == other.abbr and
+ self.isstd == other.isstd and
+ self.isgmt == other.isgmt and
+ self.dstoffset == other.dstoffset)
+
+ __hash__ = None
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ def __getstate__(self):
+ state = {}
+ for name in self.__slots__:
+ state[name] = getattr(self, name, None)
+ return state
+
+ def __setstate__(self, state):
+ for name in self.__slots__:
+ if name in state:
+ setattr(self, name, state[name])
+
+
+class _tzfile(object):
+ """
+ Lightweight class for holding the relevant transition and time zone
+ information read from binary tzfiles.
+ """
+ attrs = ['trans_list', 'trans_list_utc', 'trans_idx', 'ttinfo_list',
+ 'ttinfo_std', 'ttinfo_dst', 'ttinfo_before', 'ttinfo_first']
+
+ def __init__(self, **kwargs):
+ for attr in self.attrs:
+ setattr(self, attr, kwargs.get(attr, None))
+
+
+class tzfile(_tzinfo):
+ """
+ This is a ``tzinfo`` subclass that allows one to use the ``tzfile(5)``
+ format timezone files to extract current and historical zone information.
+
+ :param fileobj:
+ This can be an opened file stream or a file name that the time zone
+ information can be read from.
+
+ :param filename:
+ This is an optional parameter specifying the source of the time zone
+ information in the event that ``fileobj`` is a file object. If omitted
+ and ``fileobj`` is a file stream, this parameter will be set either to
+ ``fileobj``'s ``name`` attribute or to ``repr(fileobj)``.
+
+ See `Sources for Time Zone and Daylight Saving Time Data
+ `_ for more information.
+ Time zone files can be compiled from the `IANA Time Zone database files
+ `_ with the `zic time zone compiler
+ `_
+
+ .. note::
+
+ Only construct a ``tzfile`` directly if you have a specific timezone
+ file on disk that you want to read into a Python ``tzinfo`` object.
+ If you want to get a ``tzfile`` representing a specific IANA zone,
+ (e.g. ``'America/New_York'``), you should call
+ :func:`dateutil.tz.gettz` with the zone identifier.
+
+
+ **Examples:**
+
+ Using the US Eastern time zone as an example, we can see that a ``tzfile``
+ provides time zone information for the standard Daylight Saving offsets:
+
+ .. testsetup:: tzfile
+
+ from dateutil.tz import gettz
+ from datetime import datetime
+
+ .. doctest:: tzfile
+
+ >>> NYC = gettz('America/New_York')
+ >>> NYC
+ tzfile('/usr/share/zoneinfo/America/New_York')
+
+ >>> print(datetime(2016, 1, 3, tzinfo=NYC)) # EST
+ 2016-01-03 00:00:00-05:00
+
+ >>> print(datetime(2016, 7, 7, tzinfo=NYC)) # EDT
+ 2016-07-07 00:00:00-04:00
+
+
+ The ``tzfile`` structure contains a fully history of the time zone,
+ so historical dates will also have the right offsets. For example, before
+ the adoption of the UTC standards, New York used local solar mean time:
+
+ .. doctest:: tzfile
+
+ >>> print(datetime(1901, 4, 12, tzinfo=NYC)) # LMT
+ 1901-04-12 00:00:00-04:56
+
+ And during World War II, New York was on "Eastern War Time", which was a
+ state of permanent daylight saving time:
+
+ .. doctest:: tzfile
+
+ >>> print(datetime(1944, 2, 7, tzinfo=NYC)) # EWT
+ 1944-02-07 00:00:00-04:00
+
+ """
+
+ def __init__(self, fileobj, filename=None):
+ super(tzfile, self).__init__()
+
+ file_opened_here = False
+ if isinstance(fileobj, string_types):
+ self._filename = fileobj
+ fileobj = open(fileobj, 'rb')
+ file_opened_here = True
+ elif filename is not None:
+ self._filename = filename
+ elif hasattr(fileobj, "name"):
+ self._filename = fileobj.name
+ else:
+ self._filename = repr(fileobj)
+
+ if fileobj is not None:
+ if not file_opened_here:
+ fileobj = _nullcontext(fileobj)
+
+ with fileobj as file_stream:
+ tzobj = self._read_tzfile(file_stream)
+
+ self._set_tzdata(tzobj)
+
+ def _set_tzdata(self, tzobj):
+ """ Set the time zone data of this object from a _tzfile object """
+ # Copy the relevant attributes over as private attributes
+ for attr in _tzfile.attrs:
+ setattr(self, '_' + attr, getattr(tzobj, attr))
+
+ def _read_tzfile(self, fileobj):
+ out = _tzfile()
+
+ # From tzfile(5):
+ #
+ # The time zone information files used by tzset(3)
+ # begin with the magic characters "TZif" to identify
+ # them as time zone information files, followed by
+ # sixteen bytes reserved for future use, followed by
+ # six four-byte values of type long, written in a
+ # ``standard'' byte order (the high-order byte
+ # of the value is written first).
+ if fileobj.read(4).decode() != "TZif":
+ raise ValueError("magic not found")
+
+ fileobj.read(16)
+
+ (
+ # The number of UTC/local indicators stored in the file.
+ ttisgmtcnt,
+
+ # The number of standard/wall indicators stored in the file.
+ ttisstdcnt,
+
+ # The number of leap seconds for which data is
+ # stored in the file.
+ leapcnt,
+
+ # The number of "transition times" for which data
+ # is stored in the file.
+ timecnt,
+
+ # The number of "local time types" for which data
+ # is stored in the file (must not be zero).
+ typecnt,
+
+ # The number of characters of "time zone
+ # abbreviation strings" stored in the file.
+ charcnt,
+
+ ) = struct.unpack(">6l", fileobj.read(24))
+
+ # The above header is followed by tzh_timecnt four-byte
+ # values of type long, sorted in ascending order.
+ # These values are written in ``standard'' byte order.
+ # Each is used as a transition time (as returned by
+ # time(2)) at which the rules for computing local time
+ # change.
+
+ if timecnt:
+ out.trans_list_utc = list(struct.unpack(">%dl" % timecnt,
+ fileobj.read(timecnt*4)))
+ else:
+ out.trans_list_utc = []
+
+ # Next come tzh_timecnt one-byte values of type unsigned
+ # char; each one tells which of the different types of
+ # ``local time'' types described in the file is associated
+ # with the same-indexed transition time. These values
+ # serve as indices into an array of ttinfo structures that
+ # appears next in the file.
+
+ if timecnt:
+ out.trans_idx = struct.unpack(">%dB" % timecnt,
+ fileobj.read(timecnt))
+ else:
+ out.trans_idx = []
+
+ # Each ttinfo structure is written as a four-byte value
+ # for tt_gmtoff of type long, in a standard byte
+ # order, followed by a one-byte value for tt_isdst
+ # and a one-byte value for tt_abbrind. In each
+ # structure, tt_gmtoff gives the number of
+ # seconds to be added to UTC, tt_isdst tells whether
+ # tm_isdst should be set by localtime(3), and
+ # tt_abbrind serves as an index into the array of
+ # time zone abbreviation characters that follow the
+ # ttinfo structure(s) in the file.
+
+ ttinfo = []
+
+ for i in range(typecnt):
+ ttinfo.append(struct.unpack(">lbb", fileobj.read(6)))
+
+ abbr = fileobj.read(charcnt).decode()
+
+ # Then there are tzh_leapcnt pairs of four-byte
+ # values, written in standard byte order; the
+ # first value of each pair gives the time (as
+ # returned by time(2)) at which a leap second
+ # occurs; the second gives the total number of
+ # leap seconds to be applied after the given time.
+ # The pairs of values are sorted in ascending order
+ # by time.
+
+ # Not used, for now (but seek for correct file position)
+ if leapcnt:
+ fileobj.seek(leapcnt * 8, os.SEEK_CUR)
+
+ # Then there are tzh_ttisstdcnt standard/wall
+ # indicators, each stored as a one-byte value;
+ # they tell whether the transition times associated
+ # with local time types were specified as standard
+ # time or wall clock time, and are used when
+ # a time zone file is used in handling POSIX-style
+ # time zone environment variables.
+
+ if ttisstdcnt:
+ isstd = struct.unpack(">%db" % ttisstdcnt,
+ fileobj.read(ttisstdcnt))
+
+ # Finally, there are tzh_ttisgmtcnt UTC/local
+ # indicators, each stored as a one-byte value;
+ # they tell whether the transition times associated
+ # with local time types were specified as UTC or
+ # local time, and are used when a time zone file
+ # is used in handling POSIX-style time zone envi-
+ # ronment variables.
+
+ if ttisgmtcnt:
+ isgmt = struct.unpack(">%db" % ttisgmtcnt,
+ fileobj.read(ttisgmtcnt))
+
+ # Build ttinfo list
+ out.ttinfo_list = []
+ for i in range(typecnt):
+ gmtoff, isdst, abbrind = ttinfo[i]
+ gmtoff = _get_supported_offset(gmtoff)
+ tti = _ttinfo()
+ tti.offset = gmtoff
+ tti.dstoffset = datetime.timedelta(0)
+ tti.delta = datetime.timedelta(seconds=gmtoff)
+ tti.isdst = isdst
+ tti.abbr = abbr[abbrind:abbr.find('\x00', abbrind)]
+ tti.isstd = (ttisstdcnt > i and isstd[i] != 0)
+ tti.isgmt = (ttisgmtcnt > i and isgmt[i] != 0)
+ out.ttinfo_list.append(tti)
+
+ # Replace ttinfo indexes for ttinfo objects.
+ out.trans_idx = [out.ttinfo_list[idx] for idx in out.trans_idx]
+
+ # Set standard, dst, and before ttinfos. before will be
+ # used when a given time is before any transitions,
+ # and will be set to the first non-dst ttinfo, or to
+ # the first dst, if all of them are dst.
+ out.ttinfo_std = None
+ out.ttinfo_dst = None
+ out.ttinfo_before = None
+ if out.ttinfo_list:
+ if not out.trans_list_utc:
+ out.ttinfo_std = out.ttinfo_first = out.ttinfo_list[0]
+ else:
+ for i in range(timecnt-1, -1, -1):
+ tti = out.trans_idx[i]
+ if not out.ttinfo_std and not tti.isdst:
+ out.ttinfo_std = tti
+ elif not out.ttinfo_dst and tti.isdst:
+ out.ttinfo_dst = tti
+
+ if out.ttinfo_std and out.ttinfo_dst:
+ break
+ else:
+ if out.ttinfo_dst and not out.ttinfo_std:
+ out.ttinfo_std = out.ttinfo_dst
+
+ for tti in out.ttinfo_list:
+ if not tti.isdst:
+ out.ttinfo_before = tti
+ break
+ else:
+ out.ttinfo_before = out.ttinfo_list[0]
+
+ # Now fix transition times to become relative to wall time.
+ #
+ # I'm not sure about this. In my tests, the tz source file
+ # is setup to wall time, and in the binary file isstd and
+ # isgmt are off, so it should be in wall time. OTOH, it's
+ # always in gmt time. Let me know if you have comments
+ # about this.
+ lastdst = None
+ lastoffset = None
+ lastdstoffset = None
+ lastbaseoffset = None
+ out.trans_list = []
+
+ for i, tti in enumerate(out.trans_idx):
+ offset = tti.offset
+ dstoffset = 0
+
+ if lastdst is not None:
+ if tti.isdst:
+ if not lastdst:
+ dstoffset = offset - lastoffset
+
+ if not dstoffset and lastdstoffset:
+ dstoffset = lastdstoffset
+
+ tti.dstoffset = datetime.timedelta(seconds=dstoffset)
+ lastdstoffset = dstoffset
+
+ # If a time zone changes its base offset during a DST transition,
+ # then you need to adjust by the previous base offset to get the
+ # transition time in local time. Otherwise you use the current
+ # base offset. Ideally, I would have some mathematical proof of
+ # why this is true, but I haven't really thought about it enough.
+ baseoffset = offset - dstoffset
+ adjustment = baseoffset
+ if (lastbaseoffset is not None and baseoffset != lastbaseoffset
+ and tti.isdst != lastdst):
+ # The base DST has changed
+ adjustment = lastbaseoffset
+
+ lastdst = tti.isdst
+ lastoffset = offset
+ lastbaseoffset = baseoffset
+
+ out.trans_list.append(out.trans_list_utc[i] + adjustment)
+
+ out.trans_idx = tuple(out.trans_idx)
+ out.trans_list = tuple(out.trans_list)
+ out.trans_list_utc = tuple(out.trans_list_utc)
+
+ return out
+
+ def _find_last_transition(self, dt, in_utc=False):
+ # If there's no list, there are no transitions to find
+ if not self._trans_list:
+ return None
+
+ timestamp = _datetime_to_timestamp(dt)
+
+ # Find where the timestamp fits in the transition list - if the
+ # timestamp is a transition time, it's part of the "after" period.
+ trans_list = self._trans_list_utc if in_utc else self._trans_list
+ idx = bisect.bisect_right(trans_list, timestamp)
+
+ # We want to know when the previous transition was, so subtract off 1
+ return idx - 1
+
+ def _get_ttinfo(self, idx):
+ # For no list or after the last transition, default to _ttinfo_std
+ if idx is None or (idx + 1) >= len(self._trans_list):
+ return self._ttinfo_std
+
+ # If there is a list and the time is before it, return _ttinfo_before
+ if idx < 0:
+ return self._ttinfo_before
+
+ return self._trans_idx[idx]
+
+ def _find_ttinfo(self, dt):
+ idx = self._resolve_ambiguous_time(dt)
+
+ return self._get_ttinfo(idx)
+
+ def fromutc(self, dt):
+ """
+ The ``tzfile`` implementation of :py:func:`datetime.tzinfo.fromutc`.
+
+ :param dt:
+ A :py:class:`datetime.datetime` object.
+
+ :raises TypeError:
+ Raised if ``dt`` is not a :py:class:`datetime.datetime` object.
+
+ :raises ValueError:
+ Raised if this is called with a ``dt`` which does not have this
+ ``tzinfo`` attached.
+
+ :return:
+ Returns a :py:class:`datetime.datetime` object representing the
+ wall time in ``self``'s time zone.
+ """
+ # These isinstance checks are in datetime.tzinfo, so we'll preserve
+ # them, even if we don't care about duck typing.
+ if not isinstance(dt, datetime.datetime):
+ raise TypeError("fromutc() requires a datetime argument")
+
+ if dt.tzinfo is not self:
+ raise ValueError("dt.tzinfo is not self")
+
+ # First treat UTC as wall time and get the transition we're in.
+ idx = self._find_last_transition(dt, in_utc=True)
+ tti = self._get_ttinfo(idx)
+
+ dt_out = dt + datetime.timedelta(seconds=tti.offset)
+
+ fold = self.is_ambiguous(dt_out, idx=idx)
+
+ return enfold(dt_out, fold=int(fold))
+
+ def is_ambiguous(self, dt, idx=None):
+ """
+ Whether or not the "wall time" of a given datetime is ambiguous in this
+ zone.
+
+ :param dt:
+ A :py:class:`datetime.datetime`, naive or time zone aware.
+
+
+ :return:
+ Returns ``True`` if ambiguous, ``False`` otherwise.
+
+ .. versionadded:: 2.6.0
+ """
+ if idx is None:
+ idx = self._find_last_transition(dt)
+
+ # Calculate the difference in offsets from current to previous
+ timestamp = _datetime_to_timestamp(dt)
+ tti = self._get_ttinfo(idx)
+
+ if idx is None or idx <= 0:
+ return False
+
+ od = self._get_ttinfo(idx - 1).offset - tti.offset
+ tt = self._trans_list[idx] # Transition time
+
+ return timestamp < tt + od
+
+ def _resolve_ambiguous_time(self, dt):
+ idx = self._find_last_transition(dt)
+
+ # If we have no transitions, return the index
+ _fold = self._fold(dt)
+ if idx is None or idx == 0:
+ return idx
+
+ # If it's ambiguous and we're in a fold, shift to a different index.
+ idx_offset = int(not _fold and self.is_ambiguous(dt, idx))
+
+ return idx - idx_offset
+
+ def utcoffset(self, dt):
+ if dt is None:
+ return None
+
+ if not self._ttinfo_std:
+ return ZERO
+
+ return self._find_ttinfo(dt).delta
+
+ def dst(self, dt):
+ if dt is None:
+ return None
+
+ if not self._ttinfo_dst:
+ return ZERO
+
+ tti = self._find_ttinfo(dt)
+
+ if not tti.isdst:
+ return ZERO
+
+ # The documentation says that utcoffset()-dst() must
+ # be constant for every dt.
+ return tti.dstoffset
+
+ @tzname_in_python2
+ def tzname(self, dt):
+ if not self._ttinfo_std or dt is None:
+ return None
+ return self._find_ttinfo(dt).abbr
+
+ def __eq__(self, other):
+ if not isinstance(other, tzfile):
+ return NotImplemented
+ return (self._trans_list == other._trans_list and
+ self._trans_idx == other._trans_idx and
+ self._ttinfo_list == other._ttinfo_list)
+
+ __hash__ = None
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ def __repr__(self):
+ return "%s(%s)" % (self.__class__.__name__, repr(self._filename))
+
+ def __reduce__(self):
+ return self.__reduce_ex__(None)
+
+ def __reduce_ex__(self, protocol):
+ return (self.__class__, (None, self._filename), self.__dict__)
+
+
+class tzrange(tzrangebase):
+ """
+ The ``tzrange`` object is a time zone specified by a set of offsets and
+ abbreviations, equivalent to the way the ``TZ`` variable can be specified
+ in POSIX-like systems, but using Python delta objects to specify DST
+ start, end and offsets.
+
+ :param stdabbr:
+ The abbreviation for standard time (e.g. ``'EST'``).
+
+ :param stdoffset:
+ An integer or :class:`datetime.timedelta` object or equivalent
+ specifying the base offset from UTC.
+
+ If unspecified, +00:00 is used.
+
+ :param dstabbr:
+ The abbreviation for DST / "Summer" time (e.g. ``'EDT'``).
+
+ If specified, with no other DST information, DST is assumed to occur
+ and the default behavior or ``dstoffset``, ``start`` and ``end`` is
+ used. If unspecified and no other DST information is specified, it
+ is assumed that this zone has no DST.
+
+ If this is unspecified and other DST information is *is* specified,
+ DST occurs in the zone but the time zone abbreviation is left
+ unchanged.
+
+ :param dstoffset:
+ A an integer or :class:`datetime.timedelta` object or equivalent
+ specifying the UTC offset during DST. If unspecified and any other DST
+ information is specified, it is assumed to be the STD offset +1 hour.
+
+ :param start:
+ A :class:`relativedelta.relativedelta` object or equivalent specifying
+ the time and time of year that daylight savings time starts. To
+ specify, for example, that DST starts at 2AM on the 2nd Sunday in
+ March, pass:
+
+ ``relativedelta(hours=2, month=3, day=1, weekday=SU(+2))``
+
+ If unspecified and any other DST information is specified, the default
+ value is 2 AM on the first Sunday in April.
+
+ :param end:
+ A :class:`relativedelta.relativedelta` object or equivalent
+ representing the time and time of year that daylight savings time
+ ends, with the same specification method as in ``start``. One note is
+ that this should point to the first time in the *standard* zone, so if
+ a transition occurs at 2AM in the DST zone and the clocks are set back
+ 1 hour to 1AM, set the ``hours`` parameter to +1.
+
+
+ **Examples:**
+
+ .. testsetup:: tzrange
+
+ from dateutil.tz import tzrange, tzstr
+
+ .. doctest:: tzrange
+
+ >>> tzstr('EST5EDT') == tzrange("EST", -18000, "EDT")
+ True
+
+ >>> from dateutil.relativedelta import *
+ >>> range1 = tzrange("EST", -18000, "EDT")
+ >>> range2 = tzrange("EST", -18000, "EDT", -14400,
+ ... relativedelta(hours=+2, month=4, day=1,
+ ... weekday=SU(+1)),
+ ... relativedelta(hours=+1, month=10, day=31,
+ ... weekday=SU(-1)))
+ >>> tzstr('EST5EDT') == range1 == range2
+ True
+
+ """
+ def __init__(self, stdabbr, stdoffset=None,
+ dstabbr=None, dstoffset=None,
+ start=None, end=None):
+
+ global relativedelta
+ from dateutil import relativedelta
+
+ self._std_abbr = stdabbr
+ self._dst_abbr = dstabbr
+
+ try:
+ stdoffset = stdoffset.total_seconds()
+ except (TypeError, AttributeError):
+ pass
+
+ try:
+ dstoffset = dstoffset.total_seconds()
+ except (TypeError, AttributeError):
+ pass
+
+ if stdoffset is not None:
+ self._std_offset = datetime.timedelta(seconds=stdoffset)
+ else:
+ self._std_offset = ZERO
+
+ if dstoffset is not None:
+ self._dst_offset = datetime.timedelta(seconds=dstoffset)
+ elif dstabbr and stdoffset is not None:
+ self._dst_offset = self._std_offset + datetime.timedelta(hours=+1)
+ else:
+ self._dst_offset = ZERO
+
+ if dstabbr and start is None:
+ self._start_delta = relativedelta.relativedelta(
+ hours=+2, month=4, day=1, weekday=relativedelta.SU(+1))
+ else:
+ self._start_delta = start
+
+ if dstabbr and end is None:
+ self._end_delta = relativedelta.relativedelta(
+ hours=+1, month=10, day=31, weekday=relativedelta.SU(-1))
+ else:
+ self._end_delta = end
+
+ self._dst_base_offset_ = self._dst_offset - self._std_offset
+ self.hasdst = bool(self._start_delta)
+
+ def transitions(self, year):
+ """
+ For a given year, get the DST on and off transition times, expressed
+ always on the standard time side. For zones with no transitions, this
+ function returns ``None``.
+
+ :param year:
+ The year whose transitions you would like to query.
+
+ :return:
+ Returns a :class:`tuple` of :class:`datetime.datetime` objects,
+ ``(dston, dstoff)`` for zones with an annual DST transition, or
+ ``None`` for fixed offset zones.
+ """
+ if not self.hasdst:
+ return None
+
+ base_year = datetime.datetime(year, 1, 1)
+
+ start = base_year + self._start_delta
+ end = base_year + self._end_delta
+
+ return (start, end)
+
+ def __eq__(self, other):
+ if not isinstance(other, tzrange):
+ return NotImplemented
+
+ return (self._std_abbr == other._std_abbr and
+ self._dst_abbr == other._dst_abbr and
+ self._std_offset == other._std_offset and
+ self._dst_offset == other._dst_offset and
+ self._start_delta == other._start_delta and
+ self._end_delta == other._end_delta)
+
+ @property
+ def _dst_base_offset(self):
+ return self._dst_base_offset_
+
+
+@six.add_metaclass(_TzStrFactory)
+class tzstr(tzrange):
+ """
+ ``tzstr`` objects are time zone objects specified by a time-zone string as
+ it would be passed to a ``TZ`` variable on POSIX-style systems (see
+ the `GNU C Library: TZ Variable`_ for more details).
+
+ There is one notable exception, which is that POSIX-style time zones use an
+ inverted offset format, so normally ``GMT+3`` would be parsed as an offset
+ 3 hours *behind* GMT. The ``tzstr`` time zone object will parse this as an
+ offset 3 hours *ahead* of GMT. If you would like to maintain the POSIX
+ behavior, pass a ``True`` value to ``posix_offset``.
+
+ The :class:`tzrange` object provides the same functionality, but is
+ specified using :class:`relativedelta.relativedelta` objects. rather than
+ strings.
+
+ :param s:
+ A time zone string in ``TZ`` variable format. This can be a
+ :class:`bytes` (2.x: :class:`str`), :class:`str` (2.x:
+ :class:`unicode`) or a stream emitting unicode characters
+ (e.g. :class:`StringIO`).
+
+ :param posix_offset:
+ Optional. If set to ``True``, interpret strings such as ``GMT+3`` or
+ ``UTC+3`` as being 3 hours *behind* UTC rather than ahead, per the
+ POSIX standard.
+
+ .. caution::
+
+ Prior to version 2.7.0, this function also supported time zones
+ in the format:
+
+ * ``EST5EDT,4,0,6,7200,10,0,26,7200,3600``
+ * ``EST5EDT,4,1,0,7200,10,-1,0,7200,3600``
+
+ This format is non-standard and has been deprecated; this function
+ will raise a :class:`DeprecatedTZFormatWarning` until
+ support is removed in a future version.
+
+ .. _`GNU C Library: TZ Variable`:
+ https://www.gnu.org/software/libc/manual/html_node/TZ-Variable.html
+ """
+ def __init__(self, s, posix_offset=False):
+ global parser
+ from dateutil.parser import _parser as parser
+
+ self._s = s
+
+ res = parser._parsetz(s)
+ if res is None or res.any_unused_tokens:
+ raise ValueError("unknown string format")
+
+ # Here we break the compatibility with the TZ variable handling.
+ # GMT-3 actually *means* the timezone -3.
+ if res.stdabbr in ("GMT", "UTC") and not posix_offset:
+ res.stdoffset *= -1
+
+ # We must initialize it first, since _delta() needs
+ # _std_offset and _dst_offset set. Use False in start/end
+ # to avoid building it two times.
+ tzrange.__init__(self, res.stdabbr, res.stdoffset,
+ res.dstabbr, res.dstoffset,
+ start=False, end=False)
+
+ if not res.dstabbr:
+ self._start_delta = None
+ self._end_delta = None
+ else:
+ self._start_delta = self._delta(res.start)
+ if self._start_delta:
+ self._end_delta = self._delta(res.end, isend=1)
+
+ self.hasdst = bool(self._start_delta)
+
+ def _delta(self, x, isend=0):
+ from dateutil import relativedelta
+ kwargs = {}
+ if x.month is not None:
+ kwargs["month"] = x.month
+ if x.weekday is not None:
+ kwargs["weekday"] = relativedelta.weekday(x.weekday, x.week)
+ if x.week > 0:
+ kwargs["day"] = 1
+ else:
+ kwargs["day"] = 31
+ elif x.day:
+ kwargs["day"] = x.day
+ elif x.yday is not None:
+ kwargs["yearday"] = x.yday
+ elif x.jyday is not None:
+ kwargs["nlyearday"] = x.jyday
+ if not kwargs:
+ # Default is to start on first sunday of april, and end
+ # on last sunday of october.
+ if not isend:
+ kwargs["month"] = 4
+ kwargs["day"] = 1
+ kwargs["weekday"] = relativedelta.SU(+1)
+ else:
+ kwargs["month"] = 10
+ kwargs["day"] = 31
+ kwargs["weekday"] = relativedelta.SU(-1)
+ if x.time is not None:
+ kwargs["seconds"] = x.time
+ else:
+ # Default is 2AM.
+ kwargs["seconds"] = 7200
+ if isend:
+ # Convert to standard time, to follow the documented way
+ # of working with the extra hour. See the documentation
+ # of the tzinfo class.
+ delta = self._dst_offset - self._std_offset
+ kwargs["seconds"] -= delta.seconds + delta.days * 86400
+ return relativedelta.relativedelta(**kwargs)
+
+ def __repr__(self):
+ return "%s(%s)" % (self.__class__.__name__, repr(self._s))
+
+
+class _tzicalvtzcomp(object):
+ def __init__(self, tzoffsetfrom, tzoffsetto, isdst,
+ tzname=None, rrule=None):
+ self.tzoffsetfrom = datetime.timedelta(seconds=tzoffsetfrom)
+ self.tzoffsetto = datetime.timedelta(seconds=tzoffsetto)
+ self.tzoffsetdiff = self.tzoffsetto - self.tzoffsetfrom
+ self.isdst = isdst
+ self.tzname = tzname
+ self.rrule = rrule
+
+
+class _tzicalvtz(_tzinfo):
+ def __init__(self, tzid, comps=[]):
+ super(_tzicalvtz, self).__init__()
+
+ self._tzid = tzid
+ self._comps = comps
+ self._cachedate = []
+ self._cachecomp = []
+ self._cache_lock = _thread.allocate_lock()
+
+ def _find_comp(self, dt):
+ if len(self._comps) == 1:
+ return self._comps[0]
+
+ dt = dt.replace(tzinfo=None)
+
+ try:
+ with self._cache_lock:
+ return self._cachecomp[self._cachedate.index(
+ (dt, self._fold(dt)))]
+ except ValueError:
+ pass
+
+ lastcompdt = None
+ lastcomp = None
+
+ for comp in self._comps:
+ compdt = self._find_compdt(comp, dt)
+
+ if compdt and (not lastcompdt or lastcompdt < compdt):
+ lastcompdt = compdt
+ lastcomp = comp
+
+ if not lastcomp:
+ # RFC says nothing about what to do when a given
+ # time is before the first onset date. We'll look for the
+ # first standard component, or the first component, if
+ # none is found.
+ for comp in self._comps:
+ if not comp.isdst:
+ lastcomp = comp
+ break
+ else:
+ lastcomp = comp[0]
+
+ with self._cache_lock:
+ self._cachedate.insert(0, (dt, self._fold(dt)))
+ self._cachecomp.insert(0, lastcomp)
+
+ if len(self._cachedate) > 10:
+ self._cachedate.pop()
+ self._cachecomp.pop()
+
+ return lastcomp
+
+ def _find_compdt(self, comp, dt):
+ if comp.tzoffsetdiff < ZERO and self._fold(dt):
+ dt -= comp.tzoffsetdiff
+
+ compdt = comp.rrule.before(dt, inc=True)
+
+ return compdt
+
+ def utcoffset(self, dt):
+ if dt is None:
+ return None
+
+ return self._find_comp(dt).tzoffsetto
+
+ def dst(self, dt):
+ comp = self._find_comp(dt)
+ if comp.isdst:
+ return comp.tzoffsetdiff
+ else:
+ return ZERO
+
+ @tzname_in_python2
+ def tzname(self, dt):
+ return self._find_comp(dt).tzname
+
+ def __repr__(self):
+ return "" % repr(self._tzid)
+
+ __reduce__ = object.__reduce__
+
+
+class tzical(object):
+ """
+ This object is designed to parse an iCalendar-style ``VTIMEZONE`` structure
+ as set out in `RFC 5545`_ Section 4.6.5 into one or more `tzinfo` objects.
+
+ :param `fileobj`:
+ A file or stream in iCalendar format, which should be UTF-8 encoded
+ with CRLF endings.
+
+ .. _`RFC 5545`: https://tools.ietf.org/html/rfc5545
+ """
+ def __init__(self, fileobj):
+ global rrule
+ from dateutil import rrule
+
+ if isinstance(fileobj, string_types):
+ self._s = fileobj
+ # ical should be encoded in UTF-8 with CRLF
+ fileobj = open(fileobj, 'r')
+ else:
+ self._s = getattr(fileobj, 'name', repr(fileobj))
+ fileobj = _nullcontext(fileobj)
+
+ self._vtz = {}
+
+ with fileobj as fobj:
+ self._parse_rfc(fobj.read())
+
+ def keys(self):
+ """
+ Retrieves the available time zones as a list.
+ """
+ return list(self._vtz.keys())
+
+ def get(self, tzid=None):
+ """
+ Retrieve a :py:class:`datetime.tzinfo` object by its ``tzid``.
+
+ :param tzid:
+ If there is exactly one time zone available, omitting ``tzid``
+ or passing :py:const:`None` value returns it. Otherwise a valid
+ key (which can be retrieved from :func:`keys`) is required.
+
+ :raises ValueError:
+ Raised if ``tzid`` is not specified but there are either more
+ or fewer than 1 zone defined.
+
+ :returns:
+ Returns either a :py:class:`datetime.tzinfo` object representing
+ the relevant time zone or :py:const:`None` if the ``tzid`` was
+ not found.
+ """
+ if tzid is None:
+ if len(self._vtz) == 0:
+ raise ValueError("no timezones defined")
+ elif len(self._vtz) > 1:
+ raise ValueError("more than one timezone available")
+ tzid = next(iter(self._vtz))
+
+ return self._vtz.get(tzid)
+
+ def _parse_offset(self, s):
+ s = s.strip()
+ if not s:
+ raise ValueError("empty offset")
+ if s[0] in ('+', '-'):
+ signal = (-1, +1)[s[0] == '+']
+ s = s[1:]
+ else:
+ signal = +1
+ if len(s) == 4:
+ return (int(s[:2]) * 3600 + int(s[2:]) * 60) * signal
+ elif len(s) == 6:
+ return (int(s[:2]) * 3600 + int(s[2:4]) * 60 + int(s[4:])) * signal
+ else:
+ raise ValueError("invalid offset: " + s)
+
+ def _parse_rfc(self, s):
+ lines = s.splitlines()
+ if not lines:
+ raise ValueError("empty string")
+
+ # Unfold
+ i = 0
+ while i < len(lines):
+ line = lines[i].rstrip()
+ if not line:
+ del lines[i]
+ elif i > 0 and line[0] == " ":
+ lines[i-1] += line[1:]
+ del lines[i]
+ else:
+ i += 1
+
+ tzid = None
+ comps = []
+ invtz = False
+ comptype = None
+ for line in lines:
+ if not line:
+ continue
+ name, value = line.split(':', 1)
+ parms = name.split(';')
+ if not parms:
+ raise ValueError("empty property name")
+ name = parms[0].upper()
+ parms = parms[1:]
+ if invtz:
+ if name == "BEGIN":
+ if value in ("STANDARD", "DAYLIGHT"):
+ # Process component
+ pass
+ else:
+ raise ValueError("unknown component: "+value)
+ comptype = value
+ founddtstart = False
+ tzoffsetfrom = None
+ tzoffsetto = None
+ rrulelines = []
+ tzname = None
+ elif name == "END":
+ if value == "VTIMEZONE":
+ if comptype:
+ raise ValueError("component not closed: "+comptype)
+ if not tzid:
+ raise ValueError("mandatory TZID not found")
+ if not comps:
+ raise ValueError(
+ "at least one component is needed")
+ # Process vtimezone
+ self._vtz[tzid] = _tzicalvtz(tzid, comps)
+ invtz = False
+ elif value == comptype:
+ if not founddtstart:
+ raise ValueError("mandatory DTSTART not found")
+ if tzoffsetfrom is None:
+ raise ValueError(
+ "mandatory TZOFFSETFROM not found")
+ if tzoffsetto is None:
+ raise ValueError(
+ "mandatory TZOFFSETFROM not found")
+ # Process component
+ rr = None
+ if rrulelines:
+ rr = rrule.rrulestr("\n".join(rrulelines),
+ compatible=True,
+ ignoretz=True,
+ cache=True)
+ comp = _tzicalvtzcomp(tzoffsetfrom, tzoffsetto,
+ (comptype == "DAYLIGHT"),
+ tzname, rr)
+ comps.append(comp)
+ comptype = None
+ else:
+ raise ValueError("invalid component end: "+value)
+ elif comptype:
+ if name == "DTSTART":
+ # DTSTART in VTIMEZONE takes a subset of valid RRULE
+ # values under RFC 5545.
+ for parm in parms:
+ if parm != 'VALUE=DATE-TIME':
+ msg = ('Unsupported DTSTART param in ' +
+ 'VTIMEZONE: ' + parm)
+ raise ValueError(msg)
+ rrulelines.append(line)
+ founddtstart = True
+ elif name in ("RRULE", "RDATE", "EXRULE", "EXDATE"):
+ rrulelines.append(line)
+ elif name == "TZOFFSETFROM":
+ if parms:
+ raise ValueError(
+ "unsupported %s parm: %s " % (name, parms[0]))
+ tzoffsetfrom = self._parse_offset(value)
+ elif name == "TZOFFSETTO":
+ if parms:
+ raise ValueError(
+ "unsupported TZOFFSETTO parm: "+parms[0])
+ tzoffsetto = self._parse_offset(value)
+ elif name == "TZNAME":
+ if parms:
+ raise ValueError(
+ "unsupported TZNAME parm: "+parms[0])
+ tzname = value
+ elif name == "COMMENT":
+ pass
+ else:
+ raise ValueError("unsupported property: "+name)
+ else:
+ if name == "TZID":
+ if parms:
+ raise ValueError(
+ "unsupported TZID parm: "+parms[0])
+ tzid = value
+ elif name in ("TZURL", "LAST-MODIFIED", "COMMENT"):
+ pass
+ else:
+ raise ValueError("unsupported property: "+name)
+ elif name == "BEGIN" and value == "VTIMEZONE":
+ tzid = None
+ comps = []
+ invtz = True
+
+ def __repr__(self):
+ return "%s(%s)" % (self.__class__.__name__, repr(self._s))
+
+
+if sys.platform != "win32":
+ TZFILES = ["/etc/localtime", "localtime"]
+ TZPATHS = ["/usr/share/zoneinfo",
+ "/usr/lib/zoneinfo",
+ "/usr/share/lib/zoneinfo",
+ "/etc/zoneinfo"]
+else:
+ TZFILES = []
+ TZPATHS = []
+
+
+def __get_gettz():
+ tzlocal_classes = (tzlocal,)
+ if tzwinlocal is not None:
+ tzlocal_classes += (tzwinlocal,)
+
+ class GettzFunc(object):
+ """
+ Retrieve a time zone object from a string representation
+
+ This function is intended to retrieve the :py:class:`tzinfo` subclass
+ that best represents the time zone that would be used if a POSIX
+ `TZ variable`_ were set to the same value.
+
+ If no argument or an empty string is passed to ``gettz``, local time
+ is returned:
+
+ .. code-block:: python3
+
+ >>> gettz()
+ tzfile('/etc/localtime')
+
+ This function is also the preferred way to map IANA tz database keys
+ to :class:`tzfile` objects:
+
+ .. code-block:: python3
+
+ >>> gettz('Pacific/Kiritimati')
+ tzfile('/usr/share/zoneinfo/Pacific/Kiritimati')
+
+ On Windows, the standard is extended to include the Windows-specific
+ zone names provided by the operating system:
+
+ .. code-block:: python3
+
+ >>> gettz('Egypt Standard Time')
+ tzwin('Egypt Standard Time')
+
+ Passing a GNU ``TZ`` style string time zone specification returns a
+ :class:`tzstr` object:
+
+ .. code-block:: python3
+
+ >>> gettz('AEST-10AEDT-11,M10.1.0/2,M4.1.0/3')
+ tzstr('AEST-10AEDT-11,M10.1.0/2,M4.1.0/3')
+
+ :param name:
+ A time zone name (IANA, or, on Windows, Windows keys), location of
+ a ``tzfile(5)`` zoneinfo file or ``TZ`` variable style time zone
+ specifier. An empty string, no argument or ``None`` is interpreted
+ as local time.
+
+ :return:
+ Returns an instance of one of ``dateutil``'s :py:class:`tzinfo`
+ subclasses.
+
+ .. versionchanged:: 2.7.0
+
+ After version 2.7.0, any two calls to ``gettz`` using the same
+ input strings will return the same object:
+
+ .. code-block:: python3
+
+ >>> tz.gettz('America/Chicago') is tz.gettz('America/Chicago')
+ True
+
+ In addition to improving performance, this ensures that
+ `"same zone" semantics`_ are used for datetimes in the same zone.
+
+
+ .. _`TZ variable`:
+ https://www.gnu.org/software/libc/manual/html_node/TZ-Variable.html
+
+ .. _`"same zone" semantics`:
+ https://blog.ganssle.io/articles/2018/02/aware-datetime-arithmetic.html
+ """
+ def __init__(self):
+
+ self.__instances = weakref.WeakValueDictionary()
+ self.__strong_cache_size = 8
+ self.__strong_cache = OrderedDict()
+ self._cache_lock = _thread.allocate_lock()
+
+ def __call__(self, name=None):
+ with self._cache_lock:
+ rv = self.__instances.get(name, None)
+
+ if rv is None:
+ rv = self.nocache(name=name)
+ if not (name is None
+ or isinstance(rv, tzlocal_classes)
+ or rv is None):
+ # tzlocal is slightly more complicated than the other
+ # time zone providers because it depends on environment
+ # at construction time, so don't cache that.
+ #
+ # We also cannot store weak references to None, so we
+ # will also not store that.
+ self.__instances[name] = rv
+ else:
+ # No need for strong caching, return immediately
+ return rv
+
+ self.__strong_cache[name] = self.__strong_cache.pop(name, rv)
+
+ if len(self.__strong_cache) > self.__strong_cache_size:
+ self.__strong_cache.popitem(last=False)
+
+ return rv
+
+ def set_cache_size(self, size):
+ with self._cache_lock:
+ self.__strong_cache_size = size
+ while len(self.__strong_cache) > size:
+ self.__strong_cache.popitem(last=False)
+
+ def cache_clear(self):
+ with self._cache_lock:
+ self.__instances = weakref.WeakValueDictionary()
+ self.__strong_cache.clear()
+
+ @staticmethod
+ def nocache(name=None):
+ """A non-cached version of gettz"""
+ tz = None
+ if not name:
+ try:
+ name = os.environ["TZ"]
+ except KeyError:
+ pass
+ if name is None or name in ("", ":"):
+ for filepath in TZFILES:
+ if not os.path.isabs(filepath):
+ filename = filepath
+ for path in TZPATHS:
+ filepath = os.path.join(path, filename)
+ if os.path.isfile(filepath):
+ break
+ else:
+ continue
+ if os.path.isfile(filepath):
+ try:
+ tz = tzfile(filepath)
+ break
+ except (IOError, OSError, ValueError):
+ pass
+ else:
+ tz = tzlocal()
+ else:
+ try:
+ if name.startswith(":"):
+ name = name[1:]
+ except TypeError as e:
+ if isinstance(name, bytes):
+ new_msg = "gettz argument should be str, not bytes"
+ six.raise_from(TypeError(new_msg), e)
+ else:
+ raise
+ if os.path.isabs(name):
+ if os.path.isfile(name):
+ tz = tzfile(name)
+ else:
+ tz = None
+ else:
+ for path in TZPATHS:
+ filepath = os.path.join(path, name)
+ if not os.path.isfile(filepath):
+ filepath = filepath.replace(' ', '_')
+ if not os.path.isfile(filepath):
+ continue
+ try:
+ tz = tzfile(filepath)
+ break
+ except (IOError, OSError, ValueError):
+ pass
+ else:
+ tz = None
+ if tzwin is not None:
+ try:
+ tz = tzwin(name)
+ except (WindowsError, UnicodeEncodeError):
+ # UnicodeEncodeError is for Python 2.7 compat
+ tz = None
+
+ if not tz:
+ from dateutil.zoneinfo import get_zonefile_instance
+ tz = get_zonefile_instance().get(name)
+
+ if not tz:
+ for c in name:
+ # name is not a tzstr unless it has at least
+ # one offset. For short values of "name", an
+ # explicit for loop seems to be the fastest way
+ # To determine if a string contains a digit
+ if c in "0123456789":
+ try:
+ tz = tzstr(name)
+ except ValueError:
+ pass
+ break
+ else:
+ if name in ("GMT", "UTC"):
+ tz = UTC
+ elif name in time.tzname:
+ tz = tzlocal()
+ return tz
+
+ return GettzFunc()
+
+
+gettz = __get_gettz()
+del __get_gettz
+
+
+def datetime_exists(dt, tz=None):
+ """
+ Given a datetime and a time zone, determine whether or not a given datetime
+ would fall in a gap.
+
+ :param dt:
+ A :class:`datetime.datetime` (whose time zone will be ignored if ``tz``
+ is provided.)
+
+ :param tz:
+ A :class:`datetime.tzinfo` with support for the ``fold`` attribute. If
+ ``None`` or not provided, the datetime's own time zone will be used.
+
+ :return:
+ Returns a boolean value whether or not the "wall time" exists in
+ ``tz``.
+
+ .. versionadded:: 2.7.0
+ """
+ if tz is None:
+ if dt.tzinfo is None:
+ raise ValueError('Datetime is naive and no time zone provided.')
+ tz = dt.tzinfo
+
+ dt = dt.replace(tzinfo=None)
+
+ # This is essentially a test of whether or not the datetime can survive
+ # a round trip to UTC.
+ dt_rt = dt.replace(tzinfo=tz).astimezone(UTC).astimezone(tz)
+ dt_rt = dt_rt.replace(tzinfo=None)
+
+ return dt == dt_rt
+
+
+def datetime_ambiguous(dt, tz=None):
+ """
+ Given a datetime and a time zone, determine whether or not a given datetime
+ is ambiguous (i.e if there are two times differentiated only by their DST
+ status).
+
+ :param dt:
+ A :class:`datetime.datetime` (whose time zone will be ignored if ``tz``
+ is provided.)
+
+ :param tz:
+ A :class:`datetime.tzinfo` with support for the ``fold`` attribute. If
+ ``None`` or not provided, the datetime's own time zone will be used.
+
+ :return:
+ Returns a boolean value whether or not the "wall time" is ambiguous in
+ ``tz``.
+
+ .. versionadded:: 2.6.0
+ """
+ if tz is None:
+ if dt.tzinfo is None:
+ raise ValueError('Datetime is naive and no time zone provided.')
+
+ tz = dt.tzinfo
+
+ # If a time zone defines its own "is_ambiguous" function, we'll use that.
+ is_ambiguous_fn = getattr(tz, 'is_ambiguous', None)
+ if is_ambiguous_fn is not None:
+ try:
+ return tz.is_ambiguous(dt)
+ except Exception:
+ pass
+
+ # If it doesn't come out and tell us it's ambiguous, we'll just check if
+ # the fold attribute has any effect on this particular date and time.
+ dt = dt.replace(tzinfo=tz)
+ wall_0 = enfold(dt, fold=0)
+ wall_1 = enfold(dt, fold=1)
+
+ same_offset = wall_0.utcoffset() == wall_1.utcoffset()
+ same_dst = wall_0.dst() == wall_1.dst()
+
+ return not (same_offset and same_dst)
+
+
+def resolve_imaginary(dt):
+ """
+ Given a datetime that may be imaginary, return an existing datetime.
+
+ This function assumes that an imaginary datetime represents what the
+ wall time would be in a zone had the offset transition not occurred, so
+ it will always fall forward by the transition's change in offset.
+
+ .. doctest::
+
+ >>> from dateutil import tz
+ >>> from datetime import datetime
+ >>> NYC = tz.gettz('America/New_York')
+ >>> print(tz.resolve_imaginary(datetime(2017, 3, 12, 2, 30, tzinfo=NYC)))
+ 2017-03-12 03:30:00-04:00
+
+ >>> KIR = tz.gettz('Pacific/Kiritimati')
+ >>> print(tz.resolve_imaginary(datetime(1995, 1, 1, 12, 30, tzinfo=KIR)))
+ 1995-01-02 12:30:00+14:00
+
+ As a note, :func:`datetime.astimezone` is guaranteed to produce a valid,
+ existing datetime, so a round-trip to and from UTC is sufficient to get
+ an extant datetime, however, this generally "falls back" to an earlier time
+ rather than falling forward to the STD side (though no guarantees are made
+ about this behavior).
+
+ :param dt:
+ A :class:`datetime.datetime` which may or may not exist.
+
+ :return:
+ Returns an existing :class:`datetime.datetime`. If ``dt`` was not
+ imaginary, the datetime returned is guaranteed to be the same object
+ passed to the function.
+
+ .. versionadded:: 2.7.0
+ """
+ if dt.tzinfo is not None and not datetime_exists(dt):
+
+ curr_offset = (dt + datetime.timedelta(hours=24)).utcoffset()
+ old_offset = (dt - datetime.timedelta(hours=24)).utcoffset()
+
+ dt += curr_offset - old_offset
+
+ return dt
+
+
+def _datetime_to_timestamp(dt):
+ """
+ Convert a :class:`datetime.datetime` object to an epoch timestamp in
+ seconds since January 1, 1970, ignoring the time zone.
+ """
+ return (dt.replace(tzinfo=None) - EPOCH).total_seconds()
+
+
+if sys.version_info >= (3, 6):
+ def _get_supported_offset(second_offset):
+ return second_offset
+else:
+ def _get_supported_offset(second_offset):
+ # For python pre-3.6, round to full-minutes if that's not the case.
+ # Python's datetime doesn't accept sub-minute timezones. Check
+ # http://python.org/sf/1447945 or https://bugs.python.org/issue5288
+ # for some information.
+ old_offset = second_offset
+ calculated_offset = 60 * ((second_offset + 30) // 60)
+ return calculated_offset
+
+
+try:
+ # Python 3.7 feature
+ from contextlib import nullcontext as _nullcontext
+except ImportError:
+ class _nullcontext(object):
+ """
+ Class for wrapping contexts so that they are passed through in a
+ with statement.
+ """
+ def __init__(self, context):
+ self.context = context
+
+ def __enter__(self):
+ return self.context
+
+ def __exit__(*args, **kwargs):
+ pass
+
+# vim:ts=4:sw=4:et
diff --git a/venv/Lib/site-packages/dateutil/tz/win.py b/venv/Lib/site-packages/dateutil/tz/win.py
new file mode 100644
index 0000000000000000000000000000000000000000..cde07ba792c40903f0c334839140173b39fd8124
--- /dev/null
+++ b/venv/Lib/site-packages/dateutil/tz/win.py
@@ -0,0 +1,370 @@
+# -*- coding: utf-8 -*-
+"""
+This module provides an interface to the native time zone data on Windows,
+including :py:class:`datetime.tzinfo` implementations.
+
+Attempting to import this module on a non-Windows platform will raise an
+:py:obj:`ImportError`.
+"""
+# This code was originally contributed by Jeffrey Harris.
+import datetime
+import struct
+
+from six.moves import winreg
+from six import text_type
+
+try:
+ import ctypes
+ from ctypes import wintypes
+except ValueError:
+ # ValueError is raised on non-Windows systems for some horrible reason.
+ raise ImportError("Running tzwin on non-Windows system")
+
+from ._common import tzrangebase
+
+__all__ = ["tzwin", "tzwinlocal", "tzres"]
+
+ONEWEEK = datetime.timedelta(7)
+
+TZKEYNAMENT = r"SOFTWARE\Microsoft\Windows NT\CurrentVersion\Time Zones"
+TZKEYNAME9X = r"SOFTWARE\Microsoft\Windows\CurrentVersion\Time Zones"
+TZLOCALKEYNAME = r"SYSTEM\CurrentControlSet\Control\TimeZoneInformation"
+
+
+def _settzkeyname():
+ handle = winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE)
+ try:
+ winreg.OpenKey(handle, TZKEYNAMENT).Close()
+ TZKEYNAME = TZKEYNAMENT
+ except WindowsError:
+ TZKEYNAME = TZKEYNAME9X
+ handle.Close()
+ return TZKEYNAME
+
+
+TZKEYNAME = _settzkeyname()
+
+
+class tzres(object):
+ """
+ Class for accessing ``tzres.dll``, which contains timezone name related
+ resources.
+
+ .. versionadded:: 2.5.0
+ """
+ p_wchar = ctypes.POINTER(wintypes.WCHAR) # Pointer to a wide char
+
+ def __init__(self, tzres_loc='tzres.dll'):
+ # Load the user32 DLL so we can load strings from tzres
+ user32 = ctypes.WinDLL('user32')
+
+ # Specify the LoadStringW function
+ user32.LoadStringW.argtypes = (wintypes.HINSTANCE,
+ wintypes.UINT,
+ wintypes.LPWSTR,
+ ctypes.c_int)
+
+ self.LoadStringW = user32.LoadStringW
+ self._tzres = ctypes.WinDLL(tzres_loc)
+ self.tzres_loc = tzres_loc
+
+ def load_name(self, offset):
+ """
+ Load a timezone name from a DLL offset (integer).
+
+ >>> from dateutil.tzwin import tzres
+ >>> tzr = tzres()
+ >>> print(tzr.load_name(112))
+ 'Eastern Standard Time'
+
+ :param offset:
+ A positive integer value referring to a string from the tzres dll.
+
+ .. note::
+
+ Offsets found in the registry are generally of the form
+ ``@tzres.dll,-114``. The offset in this case is 114, not -114.
+
+ """
+ resource = self.p_wchar()
+ lpBuffer = ctypes.cast(ctypes.byref(resource), wintypes.LPWSTR)
+ nchar = self.LoadStringW(self._tzres._handle, offset, lpBuffer, 0)
+ return resource[:nchar]
+
+ def name_from_string(self, tzname_str):
+ """
+ Parse strings as returned from the Windows registry into the time zone
+ name as defined in the registry.
+
+ >>> from dateutil.tzwin import tzres
+ >>> tzr = tzres()
+ >>> print(tzr.name_from_string('@tzres.dll,-251'))
+ 'Dateline Daylight Time'
+ >>> print(tzr.name_from_string('Eastern Standard Time'))
+ 'Eastern Standard Time'
+
+ :param tzname_str:
+ A timezone name string as returned from a Windows registry key.
+
+ :return:
+ Returns the localized timezone string from tzres.dll if the string
+ is of the form `@tzres.dll,-offset`, else returns the input string.
+ """
+ if not tzname_str.startswith('@'):
+ return tzname_str
+
+ name_splt = tzname_str.split(',-')
+ try:
+ offset = int(name_splt[1])
+ except:
+ raise ValueError("Malformed timezone string.")
+
+ return self.load_name(offset)
+
+
+class tzwinbase(tzrangebase):
+ """tzinfo class based on win32's timezones available in the registry."""
+ def __init__(self):
+ raise NotImplementedError('tzwinbase is an abstract base class')
+
+ def __eq__(self, other):
+ # Compare on all relevant dimensions, including name.
+ if not isinstance(other, tzwinbase):
+ return NotImplemented
+
+ return (self._std_offset == other._std_offset and
+ self._dst_offset == other._dst_offset and
+ self._stddayofweek == other._stddayofweek and
+ self._dstdayofweek == other._dstdayofweek and
+ self._stdweeknumber == other._stdweeknumber and
+ self._dstweeknumber == other._dstweeknumber and
+ self._stdhour == other._stdhour and
+ self._dsthour == other._dsthour and
+ self._stdminute == other._stdminute and
+ self._dstminute == other._dstminute and
+ self._std_abbr == other._std_abbr and
+ self._dst_abbr == other._dst_abbr)
+
+ @staticmethod
+ def list():
+ """Return a list of all time zones known to the system."""
+ with winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) as handle:
+ with winreg.OpenKey(handle, TZKEYNAME) as tzkey:
+ result = [winreg.EnumKey(tzkey, i)
+ for i in range(winreg.QueryInfoKey(tzkey)[0])]
+ return result
+
+ def display(self):
+ """
+ Return the display name of the time zone.
+ """
+ return self._display
+
+ def transitions(self, year):
+ """
+ For a given year, get the DST on and off transition times, expressed
+ always on the standard time side. For zones with no transitions, this
+ function returns ``None``.
+
+ :param year:
+ The year whose transitions you would like to query.
+
+ :return:
+ Returns a :class:`tuple` of :class:`datetime.datetime` objects,
+ ``(dston, dstoff)`` for zones with an annual DST transition, or
+ ``None`` for fixed offset zones.
+ """
+
+ if not self.hasdst:
+ return None
+
+ dston = picknthweekday(year, self._dstmonth, self._dstdayofweek,
+ self._dsthour, self._dstminute,
+ self._dstweeknumber)
+
+ dstoff = picknthweekday(year, self._stdmonth, self._stddayofweek,
+ self._stdhour, self._stdminute,
+ self._stdweeknumber)
+
+ # Ambiguous dates default to the STD side
+ dstoff -= self._dst_base_offset
+
+ return dston, dstoff
+
+ def _get_hasdst(self):
+ return self._dstmonth != 0
+
+ @property
+ def _dst_base_offset(self):
+ return self._dst_base_offset_
+
+
+class tzwin(tzwinbase):
+ """
+ Time zone object created from the zone info in the Windows registry
+
+ These are similar to :py:class:`dateutil.tz.tzrange` objects in that
+ the time zone data is provided in the format of a single offset rule
+ for either 0 or 2 time zone transitions per year.
+
+ :param: name
+ The name of a Windows time zone key, e.g. "Eastern Standard Time".
+ The full list of keys can be retrieved with :func:`tzwin.list`.
+ """
+
+ def __init__(self, name):
+ self._name = name
+
+ with winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) as handle:
+ tzkeyname = text_type("{kn}\\{name}").format(kn=TZKEYNAME, name=name)
+ with winreg.OpenKey(handle, tzkeyname) as tzkey:
+ keydict = valuestodict(tzkey)
+
+ self._std_abbr = keydict["Std"]
+ self._dst_abbr = keydict["Dlt"]
+
+ self._display = keydict["Display"]
+
+ # See http://ww_winreg.jsiinc.com/SUBA/tip0300/rh0398.htm
+ tup = struct.unpack("=3l16h", keydict["TZI"])
+ stdoffset = -tup[0]-tup[1] # Bias + StandardBias * -1
+ dstoffset = stdoffset-tup[2] # + DaylightBias * -1
+ self._std_offset = datetime.timedelta(minutes=stdoffset)
+ self._dst_offset = datetime.timedelta(minutes=dstoffset)
+
+ # for the meaning see the win32 TIME_ZONE_INFORMATION structure docs
+ # http://msdn.microsoft.com/en-us/library/windows/desktop/ms725481(v=vs.85).aspx
+ (self._stdmonth,
+ self._stddayofweek, # Sunday = 0
+ self._stdweeknumber, # Last = 5
+ self._stdhour,
+ self._stdminute) = tup[4:9]
+
+ (self._dstmonth,
+ self._dstdayofweek, # Sunday = 0
+ self._dstweeknumber, # Last = 5
+ self._dsthour,
+ self._dstminute) = tup[12:17]
+
+ self._dst_base_offset_ = self._dst_offset - self._std_offset
+ self.hasdst = self._get_hasdst()
+
+ def __repr__(self):
+ return "tzwin(%s)" % repr(self._name)
+
+ def __reduce__(self):
+ return (self.__class__, (self._name,))
+
+
+class tzwinlocal(tzwinbase):
+ """
+ Class representing the local time zone information in the Windows registry
+
+ While :class:`dateutil.tz.tzlocal` makes system calls (via the :mod:`time`
+ module) to retrieve time zone information, ``tzwinlocal`` retrieves the
+ rules directly from the Windows registry and creates an object like
+ :class:`dateutil.tz.tzwin`.
+
+ Because Windows does not have an equivalent of :func:`time.tzset`, on
+ Windows, :class:`dateutil.tz.tzlocal` instances will always reflect the
+ time zone settings *at the time that the process was started*, meaning
+ changes to the machine's time zone settings during the run of a program
+ on Windows will **not** be reflected by :class:`dateutil.tz.tzlocal`.
+ Because ``tzwinlocal`` reads the registry directly, it is unaffected by
+ this issue.
+ """
+ def __init__(self):
+ with winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) as handle:
+ with winreg.OpenKey(handle, TZLOCALKEYNAME) as tzlocalkey:
+ keydict = valuestodict(tzlocalkey)
+
+ self._std_abbr = keydict["StandardName"]
+ self._dst_abbr = keydict["DaylightName"]
+
+ try:
+ tzkeyname = text_type('{kn}\\{sn}').format(kn=TZKEYNAME,
+ sn=self._std_abbr)
+ with winreg.OpenKey(handle, tzkeyname) as tzkey:
+ _keydict = valuestodict(tzkey)
+ self._display = _keydict["Display"]
+ except OSError:
+ self._display = None
+
+ stdoffset = -keydict["Bias"]-keydict["StandardBias"]
+ dstoffset = stdoffset-keydict["DaylightBias"]
+
+ self._std_offset = datetime.timedelta(minutes=stdoffset)
+ self._dst_offset = datetime.timedelta(minutes=dstoffset)
+
+ # For reasons unclear, in this particular key, the day of week has been
+ # moved to the END of the SYSTEMTIME structure.
+ tup = struct.unpack("=8h", keydict["StandardStart"])
+
+ (self._stdmonth,
+ self._stdweeknumber, # Last = 5
+ self._stdhour,
+ self._stdminute) = tup[1:5]
+
+ self._stddayofweek = tup[7]
+
+ tup = struct.unpack("=8h", keydict["DaylightStart"])
+
+ (self._dstmonth,
+ self._dstweeknumber, # Last = 5
+ self._dsthour,
+ self._dstminute) = tup[1:5]
+
+ self._dstdayofweek = tup[7]
+
+ self._dst_base_offset_ = self._dst_offset - self._std_offset
+ self.hasdst = self._get_hasdst()
+
+ def __repr__(self):
+ return "tzwinlocal()"
+
+ def __str__(self):
+ # str will return the standard name, not the daylight name.
+ return "tzwinlocal(%s)" % repr(self._std_abbr)
+
+ def __reduce__(self):
+ return (self.__class__, ())
+
+
+def picknthweekday(year, month, dayofweek, hour, minute, whichweek):
+ """ dayofweek == 0 means Sunday, whichweek 5 means last instance """
+ first = datetime.datetime(year, month, 1, hour, minute)
+
+ # This will work if dayofweek is ISO weekday (1-7) or Microsoft-style (0-6),
+ # Because 7 % 7 = 0
+ weekdayone = first.replace(day=((dayofweek - first.isoweekday()) % 7) + 1)
+ wd = weekdayone + ((whichweek - 1) * ONEWEEK)
+ if (wd.month != month):
+ wd -= ONEWEEK
+
+ return wd
+
+
+def valuestodict(key):
+ """Convert a registry key's values to a dictionary."""
+ dout = {}
+ size = winreg.QueryInfoKey(key)[1]
+ tz_res = None
+
+ for i in range(size):
+ key_name, value, dtype = winreg.EnumValue(key, i)
+ if dtype == winreg.REG_DWORD or dtype == winreg.REG_DWORD_LITTLE_ENDIAN:
+ # If it's a DWORD (32-bit integer), it's stored as unsigned - convert
+ # that to a proper signed integer
+ if value & (1 << 31):
+ value = value - (1 << 32)
+ elif dtype == winreg.REG_SZ:
+ # If it's a reference to the tzres DLL, load the actual string
+ if value.startswith('@tzres'):
+ tz_res = tz_res or tzres()
+ value = tz_res.name_from_string(value)
+
+ value = value.rstrip('\x00') # Remove trailing nulls
+
+ dout[key_name] = value
+
+ return dout
diff --git a/venv/Lib/site-packages/dateutil/tzwin.py b/venv/Lib/site-packages/dateutil/tzwin.py
new file mode 100644
index 0000000000000000000000000000000000000000..cebc673e40fc376653ebf037e96f0a6d0b33e906
--- /dev/null
+++ b/venv/Lib/site-packages/dateutil/tzwin.py
@@ -0,0 +1,2 @@
+# tzwin has moved to dateutil.tz.win
+from .tz.win import *
diff --git a/venv/Lib/site-packages/dateutil/utils.py b/venv/Lib/site-packages/dateutil/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..dd2d245a0bebcd5fc37ac20526aabbd5358dab0e
--- /dev/null
+++ b/venv/Lib/site-packages/dateutil/utils.py
@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+"""
+This module offers general convenience and utility functions for dealing with
+datetimes.
+
+.. versionadded:: 2.7.0
+"""
+from __future__ import unicode_literals
+
+from datetime import datetime, time
+
+
+def today(tzinfo=None):
+ """
+ Returns a :py:class:`datetime` representing the current day at midnight
+
+ :param tzinfo:
+ The time zone to attach (also used to determine the current day).
+
+ :return:
+ A :py:class:`datetime.datetime` object representing the current day
+ at midnight.
+ """
+
+ dt = datetime.now(tzinfo)
+ return datetime.combine(dt.date(), time(0, tzinfo=tzinfo))
+
+
+def default_tzinfo(dt, tzinfo):
+ """
+ Sets the ``tzinfo`` parameter on naive datetimes only
+
+ This is useful for example when you are provided a datetime that may have
+ either an implicit or explicit time zone, such as when parsing a time zone
+ string.
+
+ .. doctest::
+
+ >>> from dateutil.tz import tzoffset
+ >>> from dateutil.parser import parse
+ >>> from dateutil.utils import default_tzinfo
+ >>> dflt_tz = tzoffset("EST", -18000)
+ >>> print(default_tzinfo(parse('2014-01-01 12:30 UTC'), dflt_tz))
+ 2014-01-01 12:30:00+00:00
+ >>> print(default_tzinfo(parse('2014-01-01 12:30'), dflt_tz))
+ 2014-01-01 12:30:00-05:00
+
+ :param dt:
+ The datetime on which to replace the time zone
+
+ :param tzinfo:
+ The :py:class:`datetime.tzinfo` subclass instance to assign to
+ ``dt`` if (and only if) it is naive.
+
+ :return:
+ Returns an aware :py:class:`datetime.datetime`.
+ """
+ if dt.tzinfo is not None:
+ return dt
+ else:
+ return dt.replace(tzinfo=tzinfo)
+
+
+def within_delta(dt1, dt2, delta):
+ """
+ Useful for comparing two datetimes that may have a negligible difference
+ to be considered equal.
+ """
+ delta = abs(delta)
+ difference = dt1 - dt2
+ return -delta <= difference <= delta
diff --git a/venv/Lib/site-packages/dateutil/zoneinfo/__init__.py b/venv/Lib/site-packages/dateutil/zoneinfo/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..34f11ad66c88047f2c049a4cdcc937b4b78ea6d6
--- /dev/null
+++ b/venv/Lib/site-packages/dateutil/zoneinfo/__init__.py
@@ -0,0 +1,167 @@
+# -*- coding: utf-8 -*-
+import warnings
+import json
+
+from tarfile import TarFile
+from pkgutil import get_data
+from io import BytesIO
+
+from dateutil.tz import tzfile as _tzfile
+
+__all__ = ["get_zonefile_instance", "gettz", "gettz_db_metadata"]
+
+ZONEFILENAME = "dateutil-zoneinfo.tar.gz"
+METADATA_FN = 'METADATA'
+
+
+class tzfile(_tzfile):
+ def __reduce__(self):
+ return (gettz, (self._filename,))
+
+
+def getzoneinfofile_stream():
+ try:
+ return BytesIO(get_data(__name__, ZONEFILENAME))
+ except IOError as e: # TODO switch to FileNotFoundError?
+ warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))
+ return None
+
+
+class ZoneInfoFile(object):
+ def __init__(self, zonefile_stream=None):
+ if zonefile_stream is not None:
+ with TarFile.open(fileobj=zonefile_stream) as tf:
+ self.zones = {zf.name: tzfile(tf.extractfile(zf), filename=zf.name)
+ for zf in tf.getmembers()
+ if zf.isfile() and zf.name != METADATA_FN}
+ # deal with links: They'll point to their parent object. Less
+ # waste of memory
+ links = {zl.name: self.zones[zl.linkname]
+ for zl in tf.getmembers() if
+ zl.islnk() or zl.issym()}
+ self.zones.update(links)
+ try:
+ metadata_json = tf.extractfile(tf.getmember(METADATA_FN))
+ metadata_str = metadata_json.read().decode('UTF-8')
+ self.metadata = json.loads(metadata_str)
+ except KeyError:
+ # no metadata in tar file
+ self.metadata = None
+ else:
+ self.zones = {}
+ self.metadata = None
+
+ def get(self, name, default=None):
+ """
+ Wrapper for :func:`ZoneInfoFile.zones.get`. This is a convenience method
+ for retrieving zones from the zone dictionary.
+
+ :param name:
+ The name of the zone to retrieve. (Generally IANA zone names)
+
+ :param default:
+ The value to return in the event of a missing key.
+
+ .. versionadded:: 2.6.0
+
+ """
+ return self.zones.get(name, default)
+
+
+# The current API has gettz as a module function, although in fact it taps into
+# a stateful class. So as a workaround for now, without changing the API, we
+# will create a new "global" class instance the first time a user requests a
+# timezone. Ugly, but adheres to the api.
+#
+# TODO: Remove after deprecation period.
+_CLASS_ZONE_INSTANCE = []
+
+
+def get_zonefile_instance(new_instance=False):
+ """
+ This is a convenience function which provides a :class:`ZoneInfoFile`
+ instance using the data provided by the ``dateutil`` package. By default, it
+ caches a single instance of the ZoneInfoFile object and returns that.
+
+ :param new_instance:
+ If ``True``, a new instance of :class:`ZoneInfoFile` is instantiated and
+ used as the cached instance for the next call. Otherwise, new instances
+ are created only as necessary.
+
+ :return:
+ Returns a :class:`ZoneInfoFile` object.
+
+ .. versionadded:: 2.6
+ """
+ if new_instance:
+ zif = None
+ else:
+ zif = getattr(get_zonefile_instance, '_cached_instance', None)
+
+ if zif is None:
+ zif = ZoneInfoFile(getzoneinfofile_stream())
+
+ get_zonefile_instance._cached_instance = zif
+
+ return zif
+
+
+def gettz(name):
+ """
+ This retrieves a time zone from the local zoneinfo tarball that is packaged
+ with dateutil.
+
+ :param name:
+ An IANA-style time zone name, as found in the zoneinfo file.
+
+ :return:
+ Returns a :class:`dateutil.tz.tzfile` time zone object.
+
+ .. warning::
+ It is generally inadvisable to use this function, and it is only
+ provided for API compatibility with earlier versions. This is *not*
+ equivalent to ``dateutil.tz.gettz()``, which selects an appropriate
+ time zone based on the inputs, favoring system zoneinfo. This is ONLY
+ for accessing the dateutil-specific zoneinfo (which may be out of
+ date compared to the system zoneinfo).
+
+ .. deprecated:: 2.6
+ If you need to use a specific zoneinfofile over the system zoneinfo,
+ instantiate a :class:`dateutil.zoneinfo.ZoneInfoFile` object and call
+ :func:`dateutil.zoneinfo.ZoneInfoFile.get(name)` instead.
+
+ Use :func:`get_zonefile_instance` to retrieve an instance of the
+ dateutil-provided zoneinfo.
+ """
+ warnings.warn("zoneinfo.gettz() will be removed in future versions, "
+ "to use the dateutil-provided zoneinfo files, instantiate a "
+ "ZoneInfoFile object and use ZoneInfoFile.zones.get() "
+ "instead. See the documentation for details.",
+ DeprecationWarning)
+
+ if len(_CLASS_ZONE_INSTANCE) == 0:
+ _CLASS_ZONE_INSTANCE.append(ZoneInfoFile(getzoneinfofile_stream()))
+ return _CLASS_ZONE_INSTANCE[0].zones.get(name)
+
+
+def gettz_db_metadata():
+ """ Get the zonefile metadata
+
+ See `zonefile_metadata`_
+
+ :returns:
+ A dictionary with the database metadata
+
+ .. deprecated:: 2.6
+ See deprecation warning in :func:`zoneinfo.gettz`. To get metadata,
+ query the attribute ``zoneinfo.ZoneInfoFile.metadata``.
+ """
+ warnings.warn("zoneinfo.gettz_db_metadata() will be removed in future "
+ "versions, to use the dateutil-provided zoneinfo files, "
+ "ZoneInfoFile object and query the 'metadata' attribute "
+ "instead. See the documentation for details.",
+ DeprecationWarning)
+
+ if len(_CLASS_ZONE_INSTANCE) == 0:
+ _CLASS_ZONE_INSTANCE.append(ZoneInfoFile(getzoneinfofile_stream()))
+ return _CLASS_ZONE_INSTANCE[0].metadata
diff --git a/venv/Lib/site-packages/dateutil/zoneinfo/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/dateutil/zoneinfo/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..079f9b5d4fcdb063b06e67211a947d4d3600fc0a
Binary files /dev/null and b/venv/Lib/site-packages/dateutil/zoneinfo/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dateutil/zoneinfo/__pycache__/rebuild.cpython-311.pyc b/venv/Lib/site-packages/dateutil/zoneinfo/__pycache__/rebuild.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..98a578a241d3c8d42e07f3c01b95e60f0b1e2f55
Binary files /dev/null and b/venv/Lib/site-packages/dateutil/zoneinfo/__pycache__/rebuild.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dateutil/zoneinfo/dateutil-zoneinfo.tar.gz b/venv/Lib/site-packages/dateutil/zoneinfo/dateutil-zoneinfo.tar.gz
new file mode 100644
index 0000000000000000000000000000000000000000..ff62a934a5009337271c60501278a7a34913a20b
--- /dev/null
+++ b/venv/Lib/site-packages/dateutil/zoneinfo/dateutil-zoneinfo.tar.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3ea52e7b6e968de0d884df1288193596fa95b803db4f92a18279a7398004475
+size 156400
diff --git a/venv/Lib/site-packages/dateutil/zoneinfo/rebuild.py b/venv/Lib/site-packages/dateutil/zoneinfo/rebuild.py
new file mode 100644
index 0000000000000000000000000000000000000000..684c6586f091350c347f2b6150935f5214ffec27
--- /dev/null
+++ b/venv/Lib/site-packages/dateutil/zoneinfo/rebuild.py
@@ -0,0 +1,75 @@
+import logging
+import os
+import tempfile
+import shutil
+import json
+from subprocess import check_call, check_output
+from tarfile import TarFile
+
+from dateutil.zoneinfo import METADATA_FN, ZONEFILENAME
+
+
+def rebuild(filename, tag=None, format="gz", zonegroups=[], metadata=None):
+ """Rebuild the internal timezone info in dateutil/zoneinfo/zoneinfo*tar*
+
+ filename is the timezone tarball from ``ftp.iana.org/tz``.
+
+ """
+ tmpdir = tempfile.mkdtemp()
+ zonedir = os.path.join(tmpdir, "zoneinfo")
+ moduledir = os.path.dirname(__file__)
+ try:
+ with TarFile.open(filename) as tf:
+ for name in zonegroups:
+ tf.extract(name, tmpdir)
+ filepaths = [os.path.join(tmpdir, n) for n in zonegroups]
+
+ _run_zic(zonedir, filepaths)
+
+ # write metadata file
+ with open(os.path.join(zonedir, METADATA_FN), 'w') as f:
+ json.dump(metadata, f, indent=4, sort_keys=True)
+ target = os.path.join(moduledir, ZONEFILENAME)
+ with TarFile.open(target, "w:%s" % format) as tf:
+ for entry in os.listdir(zonedir):
+ entrypath = os.path.join(zonedir, entry)
+ tf.add(entrypath, entry)
+ finally:
+ shutil.rmtree(tmpdir)
+
+
+def _run_zic(zonedir, filepaths):
+ """Calls the ``zic`` compiler in a compatible way to get a "fat" binary.
+
+ Recent versions of ``zic`` default to ``-b slim``, while older versions
+ don't even have the ``-b`` option (but default to "fat" binaries). The
+ current version of dateutil does not support Version 2+ TZif files, which
+ causes problems when used in conjunction with "slim" binaries, so this
+ function is used to ensure that we always get a "fat" binary.
+ """
+
+ try:
+ help_text = check_output(["zic", "--help"])
+ except OSError as e:
+ _print_on_nosuchfile(e)
+ raise
+
+ if b"-b " in help_text:
+ bloat_args = ["-b", "fat"]
+ else:
+ bloat_args = []
+
+ check_call(["zic"] + bloat_args + ["-d", zonedir] + filepaths)
+
+
+def _print_on_nosuchfile(e):
+ """Print helpful troubleshooting message
+
+ e is an exception raised by subprocess.check_call()
+
+ """
+ if e.errno == 2:
+ logging.error(
+ "Could not find zic. Perhaps you need to install "
+ "libc-bin or some other package that provides it, "
+ "or it's not in your PATH?")
diff --git a/venv/Lib/site-packages/dill/__diff.py b/venv/Lib/site-packages/dill/__diff.py
new file mode 100644
index 0000000000000000000000000000000000000000..6fa8dd51bf87ed14939e2015d861986ad1181b2e
--- /dev/null
+++ b/venv/Lib/site-packages/dill/__diff.py
@@ -0,0 +1,234 @@
+#!/usr/bin/env python
+#
+# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
+# Copyright (c) 2008-2016 California Institute of Technology.
+# Copyright (c) 2016-2026 The Uncertainty Quantification Foundation.
+# License: 3-clause BSD. The full license text is available at:
+# - https://github.com/uqfoundation/dill/blob/master/LICENSE
+
+"""
+Module to show if an object has changed since it was memorised
+"""
+
+import builtins
+import os
+import sys
+import types
+try:
+ import numpy.ma
+ HAS_NUMPY = True
+except ImportError:
+ HAS_NUMPY = False
+
+# pypy doesn't use reference counting
+getrefcount = getattr(sys, 'getrefcount', lambda x:0)
+
+# memo of objects indexed by id to a tuple (attributes, sequence items)
+# attributes is a dict indexed by attribute name to attribute id
+# sequence items is either a list of ids, of a dictionary of keys to ids
+memo = {}
+id_to_obj = {}
+# types that cannot have changing attributes
+builtins_types = set((str, list, dict, set, frozenset, int))
+dont_memo = set(id(i) for i in (memo, sys.modules, sys.path_importer_cache,
+ os.environ, id_to_obj))
+
+
+def get_attrs(obj):
+ """
+ Gets all the attributes of an object though its __dict__ or return None
+ """
+ if type(obj) in builtins_types \
+ or type(obj) is type and obj in builtins_types:
+ return
+ return getattr(obj, '__dict__', None)
+
+
+def get_seq(obj, cache={str: False, frozenset: False, list: True, set: True,
+ dict: True, tuple: True, type: False,
+ types.ModuleType: False, types.FunctionType: False,
+ types.BuiltinFunctionType: False}):
+ """
+ Gets all the items in a sequence or return None
+ """
+ try:
+ o_type = obj.__class__
+ except AttributeError:
+ o_type = type(obj)
+ hsattr = hasattr
+ if o_type in cache:
+ if cache[o_type]:
+ if hsattr(obj, "copy"):
+ return obj.copy()
+ return obj
+ elif HAS_NUMPY and o_type in (numpy.ndarray, numpy.ma.core.MaskedConstant):
+ if obj.shape and obj.size:
+ return obj
+ else:
+ return []
+ elif hsattr(obj, "__contains__") and hsattr(obj, "__iter__") \
+ and hsattr(obj, "__len__") and hsattr(o_type, "__contains__") \
+ and hsattr(o_type, "__iter__") and hsattr(o_type, "__len__"):
+ cache[o_type] = True
+ if hsattr(obj, "copy"):
+ return obj.copy()
+ return obj
+ else:
+ cache[o_type] = False
+ return None
+
+
+def memorise(obj, force=False):
+ """
+ Adds an object to the memo, and recursively adds all the objects
+ attributes, and if it is a container, its items. Use force=True to update
+ an object already in the memo. Updating is not recursively done.
+ """
+ obj_id = id(obj)
+ if obj_id in memo and not force or obj_id in dont_memo:
+ return
+ id_ = id
+ g = get_attrs(obj)
+ if g is None:
+ attrs_id = None
+ else:
+ attrs_id = dict((key,id_(value)) for key, value in g.items())
+
+ s = get_seq(obj)
+ if s is None:
+ seq_id = None
+ elif hasattr(s, "items"):
+ seq_id = dict((id_(key),id_(value)) for key, value in s.items())
+ elif not hasattr(s, "__len__"): #XXX: avoid TypeError from unexpected case
+ seq_id = None
+ else:
+ seq_id = [id_(i) for i in s]
+
+ memo[obj_id] = attrs_id, seq_id
+ id_to_obj[obj_id] = obj
+ mem = memorise
+ if g is not None:
+ [mem(value) for key, value in g.items()]
+
+ if s is not None:
+ if hasattr(s, "items"):
+ [(mem(key), mem(item))
+ for key, item in s.items()]
+ else:
+ if hasattr(s, '__len__'):
+ [mem(item) for item in s]
+ else: mem(s)
+
+
+def release_gone():
+ itop, mp, src = id_to_obj.pop, memo.pop, getrefcount
+ [(itop(id_), mp(id_)) for id_, obj in list(id_to_obj.items())
+ if src(obj) < 4] #XXX: correct for pypy?
+
+
+def whats_changed(obj, seen=None, simple=False, first=True):
+ """
+ Check an object against the memo. Returns a list in the form
+ (attribute changes, container changed). Attribute changes is a dict of
+ attribute name to attribute value. container changed is a boolean.
+ If simple is true, just returns a boolean. None for either item means
+ that it has not been checked yet
+ """
+ # Special cases
+ if first:
+ # ignore the _ variable, which only appears in interactive sessions
+ if "_" in builtins.__dict__:
+ del builtins._
+ if seen is None:
+ seen = {}
+
+ obj_id = id(obj)
+
+ if obj_id in seen:
+ if simple:
+ return any(seen[obj_id])
+ return seen[obj_id]
+
+ # Safety checks
+ if obj_id in dont_memo:
+ seen[obj_id] = [{}, False]
+ if simple:
+ return False
+ return seen[obj_id]
+ elif obj_id not in memo:
+ if simple:
+ return True
+ else:
+ raise RuntimeError("Object not memorised " + str(obj))
+
+ seen[obj_id] = ({}, False)
+
+ chngd = whats_changed
+ id_ = id
+
+ # compare attributes
+ attrs = get_attrs(obj)
+ if attrs is None:
+ changed = {}
+ else:
+ obj_attrs = memo[obj_id][0]
+ obj_get = obj_attrs.get
+ changed = dict((key,None) for key in obj_attrs if key not in attrs)
+ for key, o in attrs.items():
+ if id_(o) != obj_get(key, None) or chngd(o, seen, True, False):
+ changed[key] = o
+
+ # compare sequence
+ items = get_seq(obj)
+ seq_diff = False
+ if (items is not None) and (hasattr(items, '__len__')):
+ obj_seq = memo[obj_id][1]
+ if (len(items) != len(obj_seq)):
+ seq_diff = True
+ elif hasattr(obj, "items"): # dict type obj
+ obj_get = obj_seq.get
+ for key, item in items.items():
+ if id_(item) != obj_get(id_(key)) \
+ or chngd(key, seen, True, False) \
+ or chngd(item, seen, True, False):
+ seq_diff = True
+ break
+ else:
+ for i, j in zip(items, obj_seq): # list type obj
+ if id_(i) != j or chngd(i, seen, True, False):
+ seq_diff = True
+ break
+ seen[obj_id] = changed, seq_diff
+ if simple:
+ return changed or seq_diff
+ return changed, seq_diff
+
+
+def has_changed(*args, **kwds):
+ kwds['simple'] = True # ignore simple if passed in
+ return whats_changed(*args, **kwds)
+
+__import__ = __import__
+
+
+def _imp(*args, **kwds):
+ """
+ Replaces the default __import__, to allow a module to be memorised
+ before the user can change it
+ """
+ before = set(sys.modules.keys())
+ mod = __import__(*args, **kwds)
+ after = set(sys.modules.keys()).difference(before)
+ for m in after:
+ memorise(sys.modules[m])
+ return mod
+
+builtins.__import__ = _imp
+if hasattr(builtins, "_"):
+ del builtins._
+
+# memorise all already imported modules. This implies that this must be
+# imported first for any changes to be recorded
+for mod in list(sys.modules.values()):
+ memorise(mod)
+release_gone()
diff --git a/venv/Lib/site-packages/dill/__info__.py b/venv/Lib/site-packages/dill/__info__.py
new file mode 100644
index 0000000000000000000000000000000000000000..87b181b1f32736cf1dbb3d6e49a729e7dbb2ceb5
--- /dev/null
+++ b/venv/Lib/site-packages/dill/__info__.py
@@ -0,0 +1,291 @@
+#!/usr/bin/env python
+#
+# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
+# Copyright (c) 2026 The Uncertainty Quantification Foundation.
+# License: 3-clause BSD. The full license text is available at:
+# - https://github.com/uqfoundation/dill/blob/master/LICENSE
+'''
+-----------------------------
+dill: serialize all of Python
+-----------------------------
+
+About Dill
+==========
+
+``dill`` extends Python's ``pickle`` module for serializing and de-serializing
+Python objects to the majority of the built-in Python types. Serialization
+is the process of converting an object to a byte stream, and the inverse
+of which is converting a byte stream back to a Python object hierarchy.
+
+``dill`` provides the user the same interface as the ``pickle`` module, and
+also includes some additional features. In addition to pickling Python
+objects, ``dill`` provides the ability to save the state of an interpreter
+session in a single command. Hence, it would be feasible to save an
+interpreter session, close the interpreter, ship the pickled file to
+another computer, open a new interpreter, unpickle the session and
+thus continue from the 'saved' state of the original interpreter
+session.
+
+``dill`` can be used to store Python objects to a file, but the primary
+usage is to send Python objects across the network as a byte stream.
+``dill`` is quite flexible, and allows arbitrary user defined classes
+and functions to be serialized. Thus ``dill`` is not intended to be
+secure against erroneously or maliciously constructed data. It is
+left to the user to decide whether the data they unpickle is from
+a trustworthy source.
+
+``dill`` is part of ``pathos``, a Python framework for heterogeneous computing.
+``dill`` is in active development, so any user feedback, bug reports, comments,
+or suggestions are highly appreciated. A list of issues is located at
+https://github.com/uqfoundation/dill/issues, with a legacy list maintained at
+https://uqfoundation.github.io/project/pathos/query.
+
+
+Major Features
+==============
+
+``dill`` can pickle the following standard types:
+
+ - none, type, bool, int, float, complex, bytes, str,
+ - tuple, list, dict, file, buffer, builtin,
+ - Python classes, namedtuples, dataclasses, metaclasses,
+ - instances of classes,
+ - set, frozenset, array, functions, exceptions
+
+``dill`` can also pickle more 'exotic' standard types:
+
+ - functions with yields, nested functions, lambdas,
+ - cell, method, unboundmethod, module, code, methodwrapper,
+ - methoddescriptor, getsetdescriptor, memberdescriptor, wrapperdescriptor,
+ - dictproxy, slice, notimplemented, ellipsis, quit
+
+``dill`` cannot yet pickle these standard types:
+
+ - frame, generator, traceback
+
+``dill`` also provides the capability to:
+
+ - save and load Python interpreter sessions
+ - save and extract the source code from functions and classes
+ - interactively diagnose pickling errors
+
+
+Current Release
+===============
+
+The latest released version of ``dill`` is available from:
+
+ https://pypi.org/project/dill
+
+``dill`` is distributed under a 3-clause BSD license.
+
+
+Development Version
+===================
+
+You can get the latest development version with all the shiny new features at:
+
+ https://github.com/uqfoundation
+
+If you have a new contribution, please submit a pull request.
+
+
+Installation
+============
+
+``dill`` can be installed with ``pip``::
+
+ $ pip install dill
+
+To optionally include the ``objgraph`` diagnostic tool in the install::
+
+ $ pip install dill[graph]
+
+To optionally include the ``gprof2dot`` diagnostic tool in the install::
+
+ $ pip install dill[profile]
+
+For windows users, to optionally install session history tools::
+
+ $ pip install dill[readline]
+
+
+Requirements
+============
+
+``dill`` requires:
+
+ - ``python`` (or ``pypy``), **>=3.9**
+ - ``setuptools``, **>=42**
+
+Optional requirements:
+
+ - ``objgraph``, **>=1.7.2**
+ - ``gprof2dot``, **>=2022.7.29**
+ - ``pyreadline``, **>=1.7.1** (on windows)
+
+
+Basic Usage
+===========
+
+``dill`` is a drop-in replacement for ``pickle``. Existing code can be
+updated to allow complete pickling using::
+
+ >>> import dill as pickle
+
+or::
+
+ >>> from dill import dumps, loads
+
+``dumps`` converts the object to a unique byte string, and ``loads`` performs
+the inverse operation::
+
+ >>> squared = lambda x: x**2
+ >>> loads(dumps(squared))(3)
+ 9
+
+There are a number of options to control serialization which are provided
+as keyword arguments to several ``dill`` functions:
+
+* with *protocol*, the pickle protocol level can be set. This uses the
+ same value as the ``pickle`` module, *DEFAULT_PROTOCOL*.
+* with *byref=True*, ``dill`` to behave a lot more like pickle with
+ certain objects (like modules) pickled by reference as opposed to
+ attempting to pickle the object itself.
+* with *recurse=True*, objects referred to in the global dictionary are
+ recursively traced and pickled, instead of the default behavior of
+ attempting to store the entire global dictionary.
+* with *fmode*, the contents of the file can be pickled along with the file
+ handle, which is useful if the object is being sent over the wire to a
+ remote system which does not have the original file on disk. Options are
+ *HANDLE_FMODE* for just the handle, *CONTENTS_FMODE* for the file content
+ and *FILE_FMODE* for content and handle.
+* with *ignore=False*, objects reconstructed with types defined in the
+ top-level script environment use the existing type in the environment
+ rather than a possibly different reconstructed type.
+
+The default serialization can also be set globally in *dill.settings*.
+Thus, we can modify how ``dill`` handles references to the global dictionary
+locally or globally::
+
+ >>> import dill.settings
+ >>> dumps(absolute) == dumps(absolute, recurse=True)
+ False
+ >>> dill.settings['recurse'] = True
+ >>> dumps(absolute) == dumps(absolute, recurse=True)
+ True
+
+``dill`` also includes source code inspection, as an alternate to pickling::
+
+ >>> import dill.source
+ >>> print(dill.source.getsource(squared))
+ squared = lambda x:x**2
+
+To aid in debugging pickling issues, use *dill.detect* which provides
+tools like pickle tracing::
+
+ >>> import dill.detect
+ >>> with dill.detect.trace():
+ >>> dumps(squared)
+ ┬ F1: at 0x7fe074f8c280>
+ ├┬ F2:
+ │└ # F2 [34 B]
+ ├┬ Co: at 0x7fe07501eb30, file "", line 1>
+ │├┬ F2:
+ ││└ # F2 [19 B]
+ │└ # Co [87 B]
+ ├┬ D1:
+ │└ # D1 [22 B]
+ ├┬ D2:
+ │└ # D2 [2 B]
+ ├┬ D2:
+ │├┬ D2:
+ ││└ # D2 [2 B]
+ │└ # D2 [23 B]
+ └ # F1 [180 B]
+
+With trace, we see how ``dill`` stored the lambda (``F1``) by first storing
+``_create_function``, the underlying code object (``Co``) and ``_create_code``
+(which is used to handle code objects), then we handle the reference to
+the global dict (``D2``) plus other dictionaries (``D1`` and ``D2``) that
+save the lambda object's state. A ``#`` marks when the object is actually stored.
+
+
+More Information
+================
+
+Probably the best way to get started is to look at the documentation at
+http://dill.rtfd.io. Also see ``dill.tests`` for a set of scripts that
+demonstrate how ``dill`` can serialize different Python objects. You can
+run the test suite with ``python -m dill.tests``. The contents of any
+pickle file can be examined with ``undill``. As ``dill`` conforms to
+the ``pickle`` interface, the examples and documentation found at
+http://docs.python.org/library/pickle.html also apply to ``dill``
+if one will ``import dill as pickle``. The source code is also generally
+well documented, so further questions may be resolved by inspecting the
+code itself. Please feel free to submit a ticket on github, or ask a
+question on stackoverflow (**@Mike McKerns**).
+If you would like to share how you use ``dill`` in your work, please send
+an email (to **mmckerns at uqfoundation dot org**).
+
+
+Citation
+========
+
+If you use ``dill`` to do research that leads to publication, we ask that you
+acknowledge use of ``dill`` by citing the following in your publication::
+
+ M.M. McKerns, L. Strand, T. Sullivan, A. Fang, M.A.G. Aivazis,
+ "Building a framework for predictive science", Proceedings of
+ the 10th Python in Science Conference, 2011;
+ http://arxiv.org/pdf/1202.1056
+
+ Michael McKerns and Michael Aivazis,
+ "pathos: a framework for heterogeneous computing", 2010- ;
+ https://uqfoundation.github.io/project/pathos
+
+Please see https://uqfoundation.github.io/project/pathos or
+http://arxiv.org/pdf/1202.1056 for further information.
+
+'''
+
+__version__ = '0.4.1'
+__author__ = 'Mike McKerns'
+
+__license__ = '''
+Copyright (c) 2004-2016 California Institute of Technology.
+Copyright (c) 2016-2026 The Uncertainty Quantification Foundation.
+All rights reserved.
+
+This software is available subject to the conditions and terms laid
+out below. By downloading and using this software you are agreeing
+to the following conditions.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+ - Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ - Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+ - Neither the names of the copyright holders nor the names of any of
+ the contributors may be used to endorse or promote products derived
+ from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''
diff --git a/venv/Lib/site-packages/dill/__init__.py b/venv/Lib/site-packages/dill/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..8c45188e68a2725216a15ed4679722a9a56a9c0d
--- /dev/null
+++ b/venv/Lib/site-packages/dill/__init__.py
@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+#
+# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
+# Copyright (c) 2008-2016 California Institute of Technology.
+# Copyright (c) 2016-2026 The Uncertainty Quantification Foundation.
+# License: 3-clause BSD. The full license text is available at:
+# - https://github.com/uqfoundation/dill/blob/master/LICENSE
+
+# author, version, license, and long description
+try: # the package is installed
+ from .__info__ import __version__, __author__, __doc__, __license__
+except: # pragma: no cover
+ import os
+ import sys
+ parent = os.path.dirname(os.path.abspath(os.path.dirname(__file__)))
+ sys.path.append(parent)
+ # get distribution meta info
+ from version import (__version__, __author__,
+ get_license_text, get_readme_as_rst)
+ __license__ = get_license_text(os.path.join(parent, 'LICENSE'))
+ __license__ = "\n%s" % __license__
+ __doc__ = get_readme_as_rst(os.path.join(parent, 'README.md'))
+ del os, sys, parent, get_license_text, get_readme_as_rst
+
+
+from ._dill import (
+ dump, dumps, load, loads, copy,
+ Pickler, Unpickler, register, pickle, pickles, check,
+ DEFAULT_PROTOCOL, HIGHEST_PROTOCOL, HANDLE_FMODE, CONTENTS_FMODE, FILE_FMODE,
+ PickleError, PickleWarning, PicklingError, PicklingWarning, UnpicklingError,
+ UnpicklingWarning,
+)
+from .session import (
+ dump_module, load_module, load_module_asdict,
+ dump_session, load_session # backward compatibility
+)
+from . import detect, logger, session, source, temp
+
+# get global settings
+from .settings import settings
+
+# make sure "trace" is turned off
+logger.trace(False)
+
+objects = {}
+# local import of dill._objects
+#from . import _objects
+#objects.update(_objects.succeeds)
+#del _objects
+
+# local import of dill.objtypes
+from . import objtypes as types
+
+def load_types(pickleable=True, unpickleable=True):
+ """load pickleable and/or unpickleable types to ``dill.types``
+
+ ``dill.types`` is meant to mimic the ``types`` module, providing a
+ registry of object types. By default, the module is empty (for import
+ speed purposes). Use the ``load_types`` function to load selected object
+ types to the ``dill.types`` module.
+
+ Args:
+ pickleable (bool, default=True): if True, load pickleable types.
+ unpickleable (bool, default=True): if True, load unpickleable types.
+
+ Returns:
+ None
+ """
+ from importlib import reload
+ # local import of dill.objects
+ from . import _objects
+ if pickleable:
+ objects.update(_objects.succeeds)
+ else:
+ [objects.pop(obj,None) for obj in _objects.succeeds]
+ if unpickleable:
+ objects.update(_objects.failures)
+ else:
+ [objects.pop(obj,None) for obj in _objects.failures]
+ objects.update(_objects.registered)
+ del _objects
+ # reset contents of types to 'empty'
+ [types.__dict__.pop(obj) for obj in list(types.__dict__.keys()) \
+ if obj.find('Type') != -1]
+ # add corresponding types from objects to types
+ reload(types)
+
+def extend(use_dill=True):
+ '''add (or remove) dill types to/from the pickle registry
+
+ by default, ``dill`` populates its types to ``pickle.Pickler.dispatch``.
+ Thus, all ``dill`` types are available upon calling ``'import pickle'``.
+ To drop all ``dill`` types from the ``pickle`` dispatch, *use_dill=False*.
+
+ Args:
+ use_dill (bool, default=True): if True, extend the dispatch table.
+
+ Returns:
+ None
+ '''
+ from ._dill import _revert_extension, _extend
+ if use_dill: _extend()
+ else: _revert_extension()
+ return
+
+extend()
+
+
+def license():
+ """print license"""
+ print (__license__)
+ return
+
+def citation():
+ """print citation"""
+ print (__doc__[-491:-118])
+ return
+
+# end of file
diff --git a/venv/Lib/site-packages/dill/__pycache__/__diff.cpython-311.pyc b/venv/Lib/site-packages/dill/__pycache__/__diff.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9fdb0fb3d6d82fab557845cfb3c7c2ee328c76e0
Binary files /dev/null and b/venv/Lib/site-packages/dill/__pycache__/__diff.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dill/__pycache__/__info__.cpython-311.pyc b/venv/Lib/site-packages/dill/__pycache__/__info__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4667b0b33ed95ec1024b833b854ca33299b6afdd
Binary files /dev/null and b/venv/Lib/site-packages/dill/__pycache__/__info__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dill/__pycache__/__init__.cpython-311.pyc b/venv/Lib/site-packages/dill/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..23db59e70a9a5b56df53feedf0bc6eb51fe79316
Binary files /dev/null and b/venv/Lib/site-packages/dill/__pycache__/__init__.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dill/__pycache__/_dill.cpython-311.pyc b/venv/Lib/site-packages/dill/__pycache__/_dill.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..2f7e683158cdf455c66a72de2a4f281c38e925ad
--- /dev/null
+++ b/venv/Lib/site-packages/dill/__pycache__/_dill.cpython-311.pyc
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5469e14d507397a9cf5c655a139202f10c09ba7d75436b734ad55a4d0f5bf5f0
+size 115170
diff --git a/venv/Lib/site-packages/dill/__pycache__/_objects.cpython-311.pyc b/venv/Lib/site-packages/dill/__pycache__/_objects.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f5b6a047778694bfe059143f3f600141fa330161
Binary files /dev/null and b/venv/Lib/site-packages/dill/__pycache__/_objects.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dill/__pycache__/_shims.cpython-311.pyc b/venv/Lib/site-packages/dill/__pycache__/_shims.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7790cb0e7389be8dfd3bb5a41f4e4e648ca8c5c4
Binary files /dev/null and b/venv/Lib/site-packages/dill/__pycache__/_shims.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dill/__pycache__/detect.cpython-311.pyc b/venv/Lib/site-packages/dill/__pycache__/detect.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8765cbae9691fc65a71fa21b872246d80e0ab454
Binary files /dev/null and b/venv/Lib/site-packages/dill/__pycache__/detect.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dill/__pycache__/logger.cpython-311.pyc b/venv/Lib/site-packages/dill/__pycache__/logger.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8bcb91a17a20e96a894f62d8462c4311068afcb6
Binary files /dev/null and b/venv/Lib/site-packages/dill/__pycache__/logger.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dill/__pycache__/objtypes.cpython-311.pyc b/venv/Lib/site-packages/dill/__pycache__/objtypes.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..662c13e2fd6b2ad97ff8c6f66509ee1136d36ac3
Binary files /dev/null and b/venv/Lib/site-packages/dill/__pycache__/objtypes.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dill/__pycache__/pointers.cpython-311.pyc b/venv/Lib/site-packages/dill/__pycache__/pointers.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c462e9684a2715d35eacc3f5b21bcac07bb2d865
Binary files /dev/null and b/venv/Lib/site-packages/dill/__pycache__/pointers.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dill/__pycache__/session.cpython-311.pyc b/venv/Lib/site-packages/dill/__pycache__/session.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3093784f8cfd6053fe6396c56151999f969025d5
Binary files /dev/null and b/venv/Lib/site-packages/dill/__pycache__/session.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dill/__pycache__/settings.cpython-311.pyc b/venv/Lib/site-packages/dill/__pycache__/settings.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..715b788d59232f96e5db83e7ff29e6fb924a075f
Binary files /dev/null and b/venv/Lib/site-packages/dill/__pycache__/settings.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dill/__pycache__/source.cpython-311.pyc b/venv/Lib/site-packages/dill/__pycache__/source.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..243ad0da1d826f0988688fed20920cdae36fcaa0
Binary files /dev/null and b/venv/Lib/site-packages/dill/__pycache__/source.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dill/__pycache__/temp.cpython-311.pyc b/venv/Lib/site-packages/dill/__pycache__/temp.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..a99cf66eb011799b3c1e8cd635572f8377bdf4d2
Binary files /dev/null and b/venv/Lib/site-packages/dill/__pycache__/temp.cpython-311.pyc differ
diff --git a/venv/Lib/site-packages/dill/_dill.py b/venv/Lib/site-packages/dill/_dill.py
new file mode 100644
index 0000000000000000000000000000000000000000..27ab56c6f7ce46d4887d0e98bbfaf05aeee56a15
--- /dev/null
+++ b/venv/Lib/site-packages/dill/_dill.py
@@ -0,0 +1,2265 @@
+# -*- coding: utf-8 -*-
+#
+# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
+# Copyright (c) 2008-2015 California Institute of Technology.
+# Copyright (c) 2016-2026 The Uncertainty Quantification Foundation.
+# License: 3-clause BSD. The full license text is available at:
+# - https://github.com/uqfoundation/dill/blob/master/LICENSE
+"""
+dill: a utility for serialization of python objects
+
+The primary functions in `dill` are :func:`dump` and
+:func:`dumps` for serialization ("pickling") to a
+file or to a string, respectively, and :func:`load`
+and :func:`loads` for deserialization ("unpickling"),
+similarly, from a file or from a string. Other notable
+functions are :func:`~dill.dump_module` and
+:func:`~dill.load_module`, which are used to save and
+restore module objects, including an intepreter session.
+
+Based on code written by Oren Tirosh and Armin Ronacher.
+Extended to a (near) full set of the builtin types (in types module),
+and coded to the pickle interface, by .
+Initial port to python3 by Jonathan Dobson, continued by mmckerns.
+Tested against "all" python types (Std. Lib. CH 1-15 @ 2.7) by mmckerns.
+Tested against CH16+ Std. Lib. ... TBD.
+"""
+
+from __future__ import annotations
+
+__all__ = [
+ 'dump','dumps','load','loads','copy',
+ 'Pickler','Unpickler','register','pickle','pickles','check',
+ 'DEFAULT_PROTOCOL','HIGHEST_PROTOCOL','HANDLE_FMODE','CONTENTS_FMODE','FILE_FMODE',
+ 'PickleError','PickleWarning','PicklingError','PicklingWarning','UnpicklingError',
+ 'UnpicklingWarning',
+]
+
+__module__ = 'dill'
+
+import warnings
+from .logger import adapter as logger
+from .logger import trace as _trace
+log = logger # backward compatibility (see issue #582)
+
+import os
+import sys
+diff = None
+_use_diff = False
+OLD38 = (sys.hexversion < 0x3080000)
+OLD39 = (sys.hexversion < 0x3090000)
+OLD310 = (sys.hexversion < 0x30a0000)
+OLD312a7 = (sys.hexversion < 0x30c00a7)
+#XXX: get types from .objtypes ?
+import builtins as __builtin__
+from pickle import _Pickler as StockPickler, Unpickler as StockUnpickler
+from pickle import GLOBAL, POP
+from _contextvars import Context as ContextType
+from _thread import LockType
+from _thread import RLock as RLockType
+try:
+ from _thread import _ExceptHookArgs as ExceptHookArgsType
+except ImportError:
+ ExceptHookArgsType = None
+try:
+ from _thread import _ThreadHandle as ThreadHandleType
+except ImportError:
+ ThreadHandleType = None
+#from io import IOBase
+from types import CodeType, FunctionType, MethodType, GeneratorType, \
+ TracebackType, FrameType, ModuleType, BuiltinMethodType
+BufferType = memoryview #XXX: unregistered
+ClassType = type # no 'old-style' classes
+EllipsisType = type(Ellipsis)
+#FileType = IOBase
+NotImplementedType = type(NotImplemented)
+SliceType = slice
+TypeType = type # 'new-style' classes #XXX: unregistered
+XRangeType = range
+from types import MappingProxyType as DictProxyType, new_class
+from pickle import DEFAULT_PROTOCOL, HIGHEST_PROTOCOL, PickleError, PicklingError, UnpicklingError
+import __main__ as _main_module
+import marshal
+import gc
+# import zlib
+import abc
+import dataclasses
+from weakref import ReferenceType, ProxyType, CallableProxyType
+from collections import OrderedDict
+from enum import Enum, EnumMeta
+from functools import partial
+from operator import itemgetter, attrgetter
+GENERATOR_FAIL = False
+import importlib.machinery
+EXTENSION_SUFFIXES = tuple(importlib.machinery.EXTENSION_SUFFIXES)
+try:
+ import ctypes
+ HAS_CTYPES = True
+ # if using `pypy`, pythonapi is not found
+ IS_PYPY = not hasattr(ctypes, 'pythonapi')
+except ImportError:
+ HAS_CTYPES = False
+ IS_PYPY = False
+NumpyUfuncType = None
+NumpyDType = None
+NumpyArrayType = None
+try:
+ if not importlib.machinery.PathFinder().find_spec('numpy'):
+ raise ImportError("No module named 'numpy'")
+ NumpyUfuncType = True
+ NumpyDType = True
+ NumpyArrayType = True
+except ImportError:
+ pass
+def __hook__():
+ global NumpyArrayType, NumpyDType, NumpyUfuncType
+ from numpy import ufunc as NumpyUfuncType
+ from numpy import ndarray as NumpyArrayType
+ from numpy import dtype as NumpyDType
+ return True
+if NumpyArrayType: # then has numpy
+ def ndarraysubclassinstance(obj_type):
+ if all((c.__module__, c.__name__) != ('numpy', 'ndarray') for c in obj_type.__mro__):
+ return False
+ # anything below here is a numpy array (or subclass) instance
+ __hook__() # import numpy (so the following works!!!)
+ # verify that __reduce__ has not been overridden
+ if obj_type.__reduce_ex__ is not NumpyArrayType.__reduce_ex__ \
+ or obj_type.__reduce__ is not NumpyArrayType.__reduce__:
+ return False
+ return True
+ def numpyufunc(obj_type):
+ return any((c.__module__, c.__name__) == ('numpy', 'ufunc') for c in obj_type.__mro__)
+ def numpydtype(obj_type):
+ if all((c.__module__, c.__name__) != ('numpy', 'dtype') for c in obj_type.__mro__):
+ return False
+ # anything below here is a numpy dtype
+ __hook__() # import numpy (so the following works!!!)
+ return obj_type is type(NumpyDType) # handles subclasses
+else:
+ def ndarraysubclassinstance(obj): return False
+ def numpyufunc(obj): return False
+ def numpydtype(obj): return False
+
+from types import GetSetDescriptorType, ClassMethodDescriptorType, \
+ WrapperDescriptorType, MethodDescriptorType, MemberDescriptorType, \
+ MethodWrapperType #XXX: unused
+
+# make sure to add these 'hand-built' types to _typemap
+CellType = type((lambda x: lambda y: x)(0).__closure__[0])
+PartialType = type(partial(int, base=2))
+SuperType = type(super(Exception, TypeError()))
+ItemGetterType = type(itemgetter(0))
+AttrGetterType = type(attrgetter('__repr__'))
+
+try:
+ from functools import _lru_cache_wrapper as LRUCacheType
+except ImportError:
+ LRUCacheType = None
+
+if not isinstance(LRUCacheType, type):
+ LRUCacheType = None
+
+def get_file_type(*args, **kwargs):
+ open = kwargs.pop("open", __builtin__.open)
+ f = open(os.devnull, *args, **kwargs)
+ t = type(f)
+ f.close()
+ return t
+
+IS_PYODIDE = sys.platform == 'emscripten'
+
+FileType = get_file_type('rb', buffering=0)
+TextWrapperType = get_file_type('r', buffering=-1)
+BufferedRandomType = None if IS_PYODIDE else get_file_type('r+b', buffering=-1)
+BufferedReaderType = get_file_type('rb', buffering=-1)
+BufferedWriterType = get_file_type('wb', buffering=-1)
+try:
+ from _pyio import open as _open
+ PyTextWrapperType = get_file_type('r', buffering=-1, open=_open)
+ PyBufferedRandomType = None if IS_PYODIDE else get_file_type('r+b', buffering=-1, open=_open)
+ PyBufferedReaderType = get_file_type('rb', buffering=-1, open=_open)
+ PyBufferedWriterType = get_file_type('wb', buffering=-1, open=_open)
+except ImportError:
+ PyTextWrapperType = PyBufferedRandomType = PyBufferedReaderType = PyBufferedWriterType = None
+from io import BytesIO as StringIO
+InputType = OutputType = None
+from socket import socket as SocketType
+#FIXME: additionally calls ForkingPickler.register several times
+from multiprocessing.reduction import _reduce_socket as reduce_socket
+try: #pragma: no cover
+ IS_IPYTHON = __IPYTHON__ # is True
+ ExitType = None # IPython.core.autocall.ExitAutocall
+ IPYTHON_SINGLETONS = ('exit', 'quit', 'get_ipython')
+except NameError:
+ IS_IPYTHON = False
+ try: ExitType = type(exit) # apparently 'exit' can be removed
+ except NameError: ExitType = None
+ IPYTHON_SINGLETONS = ()
+
+import inspect
+import typing
+
+
+### Shims for different versions of Python and dill
+class Sentinel(object):
+ """
+ Create a unique sentinel object that is pickled as a constant.
+ """
+ def __init__(self, name, module_name=None):
+ self.name = name
+ if module_name is None:
+ # Use the calling frame's module
+ self.__module__ = inspect.currentframe().f_back.f_globals['__name__']
+ else:
+ self.__module__ = module_name # pragma: no cover
+ def __repr__(self):
+ return self.__module__ + '.' + self.name # pragma: no cover
+ def __copy__(self):
+ return self # pragma: no cover
+ def __deepcopy__(self, memo):
+ return self # pragma: no cover
+ def __reduce__(self):
+ return self.name
+ def __reduce_ex__(self, protocol):
+ return self.name
+
+from . import _shims
+from ._shims import Reduce, Getattr
+
+### File modes
+#: Pickles the file handle, preserving mode. The position of the unpickled
+#: object is as for a new file handle.
+HANDLE_FMODE = 0
+#: Pickles the file contents, creating a new file if on load the file does
+#: not exist. The position = min(pickled position, EOF) and mode is chosen
+#: as such that "best" preserves behavior of the original file.
+CONTENTS_FMODE = 1
+#: Pickles the entire file (handle and contents), preserving mode and position.
+FILE_FMODE = 2
+
+### Shorthands (modified from python2.5/lib/pickle.py)
+def copy(obj, *args, **kwds):
+ """
+ Use pickling to 'copy' an object (i.e. `loads(dumps(obj))`).
+
+ See :func:`dumps` and :func:`loads` for keyword arguments.
+ """
+ ignore = kwds.pop('ignore', Unpickler.settings['ignore'])
+ return loads(dumps(obj, *args, **kwds), ignore=ignore)
+
+def dump(obj, file, protocol=None, byref=None, fmode=None, recurse=None, **kwds):#, strictio=None):
+ """
+ Pickle an object to a file.
+
+ See :func:`dumps` for keyword arguments.
+ """
+ from .settings import settings
+ protocol = settings['protocol'] if protocol is None else int(protocol)
+ _kwds = kwds.copy()
+ _kwds.update(dict(byref=byref, fmode=fmode, recurse=recurse))
+ Pickler(file, protocol, **_kwds).dump(obj)
+ return
+
+def dumps(obj, protocol=None, byref=None, fmode=None, recurse=None, **kwds):#, strictio=None):
+ """
+ Pickle an object to a string.
+
+ *protocol* is the pickler protocol, as defined for Python *pickle*.
+
+ If *byref=True*, then dill behaves a lot more like pickle as certain
+ objects (like modules) are pickled by reference as opposed to attempting
+ to pickle the object itself.
+
+ If *recurse=True*, then objects referred to in the global dictionary
+ are recursively traced and pickled, instead of the default behavior
+ of attempting to store the entire global dictionary. This is needed for
+ functions defined via *exec()*.
+
+ *fmode* (:const:`HANDLE_FMODE`, :const:`CONTENTS_FMODE`,
+ or :const:`FILE_FMODE`) indicates how file handles will be pickled.
+ For example, when pickling a data file handle for transfer to a remote
+ compute service, *FILE_FMODE* will include the file contents in the
+ pickle and cursor position so that a remote method can operate
+ transparently on an object with an open file handle.
+
+ Default values for keyword arguments can be set in :mod:`dill.settings`.
+ """
+ file = StringIO()
+ dump(obj, file, protocol, byref, fmode, recurse, **kwds)#, strictio)
+ return file.getvalue()
+
+def load(file, ignore=None, **kwds):
+ """
+ Unpickle an object from a file.
+
+ See :func:`loads` for keyword arguments.
+ """
+ return Unpickler(file, ignore=ignore, **kwds).load()
+
+def loads(str, ignore=None, **kwds):
+ """
+ Unpickle an object from a string.
+
+ If *ignore=False* then objects whose class is defined in the module
+ *__main__* are updated to reference the existing class in *__main__*,
+ otherwise they are left to refer to the reconstructed type, which may
+ be different.
+
+ Default values for keyword arguments can be set in :mod:`dill.settings`.
+ """
+ file = StringIO(str)
+ return load(file, ignore, **kwds)
+
+# def dumpzs(obj, protocol=None):
+# """pickle an object to a compressed string"""
+# return zlib.compress(dumps(obj, protocol))
+
+# def loadzs(str):
+# """unpickle an object from a compressed string"""
+# return loads(zlib.decompress(str))
+
+### End: Shorthands ###
+
+class MetaCatchingDict(dict):
+ def get(self, key, default=None):
+ try:
+ return self[key]
+ except KeyError:
+ return default
+
+ def __missing__(self, key):
+ if issubclass(key, type):
+ return save_type
+ else:
+ raise KeyError()
+
+class PickleWarning(Warning, PickleError):
+ pass
+
+class PicklingWarning(PickleWarning, PicklingError):
+ pass
+
+class UnpicklingWarning(PickleWarning, UnpicklingError):
+ pass
+
+### Extend the Picklers
+class Pickler(StockPickler):
+ """python's Pickler extended to interpreter sessions"""
+ dispatch: typing.Dict[type, typing.Callable[[Pickler, typing.Any], None]] \
+ = MetaCatchingDict(StockPickler.dispatch.copy())
+ """The dispatch table, a dictionary of serializing functions used
+ by Pickler to save objects of specific types. Use :func:`pickle`
+ or :func:`register` to associate types to custom functions.
+
+ :meta hide-value:
+ """
+ _session = False
+ from .settings import settings
+
+ def __init__(self, file, *args, **kwds):
+ settings = Pickler.settings
+ _byref = kwds.pop('byref', None)
+ #_strictio = kwds.pop('strictio', None)
+ _fmode = kwds.pop('fmode', None)
+ _recurse = kwds.pop('recurse', None)
+ StockPickler.__init__(self, file, *args, **kwds)
+ self._main = _main_module
+ self._diff_cache = {}
+ self._byref = settings['byref'] if _byref is None else _byref
+ self._strictio = False #_strictio
+ self._fmode = settings['fmode'] if _fmode is None else _fmode
+ self._recurse = settings['recurse'] if _recurse is None else _recurse
+ self._postproc = OrderedDict()
+ self._file = file
+
+ def save(self, obj, save_persistent_id=True):
+ # numpy hack
+ obj_type = type(obj)
+ if NumpyArrayType and not (obj_type is type or obj_type in Pickler.dispatch):
+ # register if the object is a numpy ufunc
+ # thanks to Paul Kienzle for pointing out ufuncs didn't pickle
+ if numpyufunc(obj_type):
+ @register(obj_type)
+ def save_numpy_ufunc(pickler, obj):
+ logger.trace(pickler, "Nu: %s", obj)
+ name = getattr(obj, '__qualname__', getattr(obj, '__name__', None))
+ StockPickler.save_global(pickler, obj, name=name)
+ logger.trace(pickler, "# Nu")
+ return
+ # NOTE: the above 'save' performs like:
+ # import copy_reg
+ # def udump(f): return f.__name__
+ # def uload(name): return getattr(numpy, name)
+ # copy_reg.pickle(NumpyUfuncType, udump, uload)
+ # register if the object is a numpy dtype
+ if numpydtype(obj_type):
+ @register(obj_type)
+ def save_numpy_dtype(pickler, obj):
+ logger.trace(pickler, "Dt: %s", obj)
+ pickler.save_reduce(_create_dtypemeta, (obj.type,), obj=obj)
+ logger.trace(pickler, "# Dt")
+ return
+ # NOTE: the above 'save' performs like:
+ # import copy_reg
+ # def uload(name): return type(NumpyDType(name))
+ # def udump(f): return uload, (f.type,)
+ # copy_reg.pickle(NumpyDTypeType, udump, uload)
+ # register if the object is a subclassed numpy array instance
+ if ndarraysubclassinstance(obj_type):
+ @register(obj_type)
+ def save_numpy_array(pickler, obj):
+ logger.trace(pickler, "Nu: (%s, %s)", obj.shape, obj.dtype)
+ npdict = getattr(obj, '__dict__', None)
+ f, args, state = obj.__reduce__()
+ pickler.save_reduce(_create_array, (f,args,state,npdict), obj=obj)
+ logger.trace(pickler, "# Nu")
+ return
+ # end numpy hack
+
+ if GENERATOR_FAIL and obj_type is GeneratorType:
+ msg = "Can't pickle %s: attribute lookup builtins.generator failed" % GeneratorType
+ raise PicklingError(msg)
+ StockPickler.save(self, obj, save_persistent_id)
+
+ save.__doc__ = StockPickler.save.__doc__
+
+ def dump(self, obj): #NOTE: if settings change, need to update attributes
+ logger.trace_setup(self)
+ StockPickler.dump(self, obj)
+ dump.__doc__ = StockPickler.dump.__doc__
+
+class Unpickler(StockUnpickler):
+ """python's Unpickler extended to interpreter sessions and more types"""
+ from .settings import settings
+ _session = False
+
+ def find_class(self, module, name):
+ if (module, name) == ('__builtin__', '__main__'):
+ return self._main.__dict__ #XXX: above set w/save_module_dict
+ elif (module, name) == ('__builtin__', 'NoneType'):
+ return type(None) #XXX: special case: NoneType missing
+ if module == 'dill.dill': module = 'dill._dill'
+ return StockUnpickler.find_class(self, module, name)
+
+ def __init__(self, *args, **kwds):
+ settings = Pickler.settings
+ _ignore = kwds.pop('ignore', None)
+ StockUnpickler.__init__(self, *args, **kwds)
+ self._main = _main_module
+ self._ignore = settings['ignore'] if _ignore is None else _ignore
+
+ def load(self): #NOTE: if settings change, need to update attributes
+ obj = StockUnpickler.load(self)
+ if type(obj).__module__ == getattr(_main_module, '__name__', '__main__'):
+ if not self._ignore:
+ # point obj class to main
+ try: obj.__class__ = getattr(self._main, type(obj).__name__)
+ except (AttributeError,TypeError): pass # defined in a file
+ #_main_module.__dict__.update(obj.__dict__) #XXX: should update globals ?
+ return obj
+ load.__doc__ = StockUnpickler.load.__doc__
+ pass
+
+'''
+def dispatch_table():
+ """get the dispatch table of registered types"""
+ return Pickler.dispatch
+'''
+
+pickle_dispatch_copy = StockPickler.dispatch.copy()
+
+def pickle(t, func):
+ """expose :attr:`~Pickler.dispatch` table for user-created extensions"""
+ Pickler.dispatch[t] = func
+ return
+
+def register(t):
+ """decorator to register types to Pickler's :attr:`~Pickler.dispatch` table"""
+ def proxy(func):
+ Pickler.dispatch[t] = func
+ return func
+ return proxy
+
+def _revert_extension():
+ """drop dill-registered types from pickle's dispatch table"""
+ for type, func in list(StockPickler.dispatch.items()):
+ if func.__module__ == __name__:
+ del StockPickler.dispatch[type]
+ if type in pickle_dispatch_copy:
+ StockPickler.dispatch[type] = pickle_dispatch_copy[type]
+
+def use_diff(on=True):
+ """
+ Reduces size of pickles by only including object which have changed.
+
+ Decreases pickle size but increases CPU time needed.
+ Also helps avoid some unpickleable objects.
+ MUST be called at start of script, otherwise changes will not be recorded.
+ """
+ global _use_diff, diff
+ _use_diff = on
+ if _use_diff and diff is None:
+ try:
+ from . import diff as d
+ except ImportError:
+ import diff as d
+ diff = d
+
+def _create_typemap():
+ import types
+ d = dict(list(__builtin__.__dict__.items()) + \
+ list(types.__dict__.items())).items()
+ for key, value in d:
+ if getattr(value, '__module__', None) == 'builtins' \
+ and type(value) is type:
+ yield key, value
+ return
+_reverse_typemap = dict(_create_typemap())
+_reverse_typemap.update({
+ 'PartialType': PartialType,
+ 'SuperType': SuperType,
+ 'ItemGetterType': ItemGetterType,
+ 'AttrGetterType': AttrGetterType,
+})
+if sys.hexversion < 0x30800a2:
+ _reverse_typemap.update({
+ 'CellType': CellType,
+ })
+
+# "Incidental" implementation specific types. Unpickling these types in another
+# implementation of Python (PyPy -> CPython) is not guaranteed to work
+
+# This dictionary should contain all types that appear in Python implementations
+# but are not defined in https://docs.python.org/3/library/types.html#standard-interpreter-types
+x=OrderedDict()
+_incedental_reverse_typemap = {
+ 'FileType': FileType,
+ 'BufferedRandomType': BufferedRandomType,
+ 'BufferedReaderType': BufferedReaderType,
+ 'BufferedWriterType': BufferedWriterType,
+ 'TextWrapperType': TextWrapperType,
+ 'PyBufferedRandomType': PyBufferedRandomType,
+ 'PyBufferedReaderType': PyBufferedReaderType,
+ 'PyBufferedWriterType': PyBufferedWriterType,
+ 'PyTextWrapperType': PyTextWrapperType,
+}
+
+_incedental_reverse_typemap.update({
+ "DictKeysType": type({}.keys()),
+ "DictValuesType": type({}.values()),
+ "DictItemsType": type({}.items()),
+
+ "OdictKeysType": type(x.keys()),
+ "OdictValuesType": type(x.values()),
+ "OdictItemsType": type(x.items()),
+})
+
+if ExitType:
+ _incedental_reverse_typemap['ExitType'] = ExitType
+if InputType:
+ _incedental_reverse_typemap['InputType'] = InputType
+ _incedental_reverse_typemap['OutputType'] = OutputType
+
+'''
+try:
+ import symtable
+ _incedental_reverse_typemap["SymtableEntryType"] = type(symtable.symtable("", "string", "exec")._table)
+except: #FIXME: fails to pickle
+ pass
+
+if sys.hexversion >= 0x30a00a0:
+ _incedental_reverse_typemap['LineIteratorType'] = type(compile('3', '', 'eval').co_lines())
+'''
+
+if sys.hexversion >= 0x30b00b0 and not IS_PYPY:
+ from types import GenericAlias
+ _incedental_reverse_typemap["GenericAliasIteratorType"] = type(iter(GenericAlias(list, (int,))))
+ '''
+ _incedental_reverse_typemap['PositionsIteratorType'] = type(compile('3', '', 'eval').co_positions())
+ '''
+
+try:
+ import winreg
+ _incedental_reverse_typemap["HKEYType"] = winreg.HKEYType
+except ImportError:
+ pass
+
+_reverse_typemap.update(_incedental_reverse_typemap)
+_incedental_types = set(_incedental_reverse_typemap.values())
+
+del x
+
+_typemap = dict((v, k) for k, v in _reverse_typemap.items())
+
+def _unmarshal(string):
+ return marshal.loads(string)
+
+def _load_type(name):
+ return _reverse_typemap[name]
+
+def _create_type(typeobj, *args):
+ return typeobj(*args)
+
+def _create_function(fcode, fglobals, fname=None, fdefaults=None,
+ fclosure=None, fdict=None, fkwdefaults=None):
+ # same as FunctionType, but enable passing __dict__ to new function,
+ # __dict__ is the storehouse for attributes added after function creation
+ func = FunctionType(fcode, fglobals or dict(), fname, fdefaults, fclosure)
+ if fdict is not None:
+ func.__dict__.update(fdict) #XXX: better copy? option to copy?
+ if fkwdefaults is not None:
+ func.__kwdefaults__ = fkwdefaults
+ # 'recurse' only stores referenced modules/objects in fglobals,
+ # thus we need to make sure that we have __builtins__ as well
+ if "__builtins__" not in func.__globals__:
+ func.__globals__["__builtins__"] = globals()["__builtins__"]
+ # assert id(fglobals) == id(func.__globals__)
+ return func
+
+class match:
+ """
+ Make avaialable a limited structural pattern matching-like syntax for Python < 3.10
+
+ Patterns can be only tuples (without types) currently.
+ Inspired by the package pattern-matching-PEP634.
+
+ Usage:
+ >>> with match(args) as m:
+ >>> if m.case(('x', 'y')):
+ >>> # use m.x and m.y
+ >>> elif m.case(('x', 'y', 'z')):
+ >>> # use m.x, m.y and m.z
+
+ Equivalent native code for Python >= 3.10:
+ >>> match args:
+ >>> case (x, y):
+ >>> # use x and y
+ >>> case (x, y, z):
+ >>> # use x, y and z
+ """
+ def __init__(self, value):
+ self.value = value
+ self._fields = None
+ def __enter__(self):
+ return self
+ def __exit__(self, *exc_info):
+ return False
+ def case(self, args): # *args, **kwargs):
+ """just handles tuple patterns"""
+ if len(self.value) != len(args): # + len(kwargs):
+ return False
+ #if not all(isinstance(arg, pat) for arg, pat in zip(self.value[len(args):], kwargs.values())):
+ # return False
+ self.args = args # (*args, *kwargs)
+ return True
+ @property
+ def fields(self):
+ # Only bind names to values if necessary.
+ if self._fields is None:
+ self._fields = dict(zip(self.args, self.value))
+ return self._fields
+ def __getattr__(self, item):
+ return self.fields[item]
+
+ALL_CODE_PARAMS = [
+ # Version New attribute CodeType parameters
+ ((3,11,'a'), 'co_endlinetable', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name qualname firstlineno linetable endlinetable columntable exceptiontable freevars cellvars'),
+ ((3,11), 'co_exceptiontable', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name qualname firstlineno linetable exceptiontable freevars cellvars'),
+ ((3,11,'p'), 'co_qualname', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name qualname firstlineno linetable freevars cellvars'),
+ ((3,10), 'co_linetable', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name firstlineno linetable freevars cellvars'),
+ ((3,8), 'co_posonlyargcount', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name firstlineno lnotab freevars cellvars'),
+ ((3,7), 'co_kwonlyargcount', 'argcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name firstlineno lnotab freevars cellvars'),
+ ]
+for version, new_attr, params in ALL_CODE_PARAMS:
+ if hasattr(CodeType, new_attr):
+ CODE_VERSION = version
+ CODE_PARAMS = params.split()
+ break
+ENCODE_PARAMS = set(CODE_PARAMS).intersection(
+ ['code', 'lnotab', 'linetable', 'endlinetable', 'columntable', 'exceptiontable'])
+
+def _create_code(*args):
+ if not isinstance(args[0], int): # co_lnotab stored from >= 3.10
+ LNOTAB, *args = args
+ else: # from < 3.10 (or pre-LNOTAB storage)
+ LNOTAB = b''
+
+ with match(args) as m:
+ # Python 3.11/3.12a (18 members)
+ if m.case((
+ 'argcount', 'posonlyargcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:6]
+ 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'qualname', 'firstlineno', # args[6:14]
+ 'linetable', 'exceptiontable', 'freevars', 'cellvars' # args[14:]
+ )):
+ if CODE_VERSION == (3,11):
+ return CodeType(
+ *args[:6],
+ args[6].encode() if hasattr(args[6], 'encode') else args[6], # code
+ *args[7:14],
+ args[14].encode() if hasattr(args[14], 'encode') else args[14], # linetable
+ args[15].encode() if hasattr(args[15], 'encode') else args[15], # exceptiontable
+ args[16],
+ args[17],
+ )
+ fields = m.fields
+ # PyPy 3.11 7.3.19+ (17 members)
+ elif m.case((
+ 'argcount', 'posonlyargcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:6]
+ 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'qualname', # args[6:13]
+ 'firstlineno', 'linetable', 'freevars', 'cellvars' # args[13:]
+ )):
+ if CODE_VERSION == (3,11,'p'):
+ return CodeType(
+ *args[:6],
+ args[6].encode() if hasattr(args[6], 'encode') else args[6], # code
+ *args[7:14],
+ args[14].encode() if hasattr(args[14], 'encode') else args[14], # linetable
+ args[15],
+ args[16],
+ )
+ fields = m.fields
+ # Python 3.10 or 3.8/3.9 (16 members)
+ elif m.case((
+ 'argcount', 'posonlyargcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:6]
+ 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'firstlineno', # args[6:13]
+ 'LNOTAB_OR_LINETABLE', 'freevars', 'cellvars' # args[13:]
+ )):
+ if CODE_VERSION == (3,10) or CODE_VERSION == (3,8):
+ return CodeType(
+ *args[:6],
+ args[6].encode() if hasattr(args[6], 'encode') else args[6], # code
+ *args[7:13],
+ args[13].encode() if hasattr(args[13], 'encode') else args[13], # lnotab/linetable
+ args[14],
+ args[15],
+ )
+ fields = m.fields
+ if CODE_VERSION >= (3,10):
+ fields['linetable'] = m.LNOTAB_OR_LINETABLE
+ else:
+ fields['lnotab'] = LNOTAB if LNOTAB else m.LNOTAB_OR_LINETABLE
+ # Python 3.7 (15 args)
+ elif m.case((
+ 'argcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:5]
+ 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'firstlineno', # args[5:12]
+ 'lnotab', 'freevars', 'cellvars' # args[12:]
+ )):
+ if CODE_VERSION == (3,7):
+ return CodeType(
+ *args[:5],
+ args[5].encode() if hasattr(args[5], 'encode') else args[5], # code
+ *args[6:12],
+ args[12].encode() if hasattr(args[12], 'encode') else args[12], # lnotab
+ args[13],
+ args[14],
+ )
+ fields = m.fields
+ # Python 3.11a (20 members)
+ elif m.case((
+ 'argcount', 'posonlyargcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:6]
+ 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'qualname', 'firstlineno', # args[6:14]
+ 'linetable', 'endlinetable', 'columntable', 'exceptiontable', 'freevars', 'cellvars' # args[14:]
+ )):
+ if CODE_VERSION == (3,11,'a'):
+ return CodeType(
+ *args[:6],
+ args[6].encode() if hasattr(args[6], 'encode') else args[6], # code
+ *args[7:14],
+ *(a.encode() if hasattr(a, 'encode') else a for a in args[14:18]), # linetable-exceptiontable
+ args[18],
+ args[19],
+ )
+ fields = m.fields
+ else:
+ raise UnpicklingError("pattern match for code object failed")
+
+ # The args format doesn't match this version.
+ fields.setdefault('posonlyargcount', 0) # from python <= 3.7
+ fields.setdefault('lnotab', LNOTAB) # from python >= 3.10
+ fields.setdefault('linetable', b'') # from python <= 3.9
+ fields.setdefault('qualname', fields['name']) # from python <= 3.10
+ fields.setdefault('exceptiontable', b'') # from python <= 3.10
+ fields.setdefault('endlinetable', None) # from python != 3.11a
+ fields.setdefault('columntable', None) # from python != 3.11a
+
+ args = (fields[k].encode() if k in ENCODE_PARAMS and hasattr(fields[k], 'encode') else fields[k]
+ for k in CODE_PARAMS)
+ return CodeType(*args)
+
+def _create_ftype(ftypeobj, func, args, kwds):
+ if kwds is None:
+ kwds = {}
+ if args is None:
+ args = ()
+ return ftypeobj(func, *args, **kwds)
+
+def _create_typing_tuple(argz, *args): #NOTE: workaround python/cpython#94245
+ if not argz:
+ return typing.Tuple[()].copy_with(())
+ if argz == ((),):
+ return typing.Tuple[()]
+ return typing.Tuple[argz]
+
+if ThreadHandleType:
+ def _create_thread_handle(ident, done, *args): #XXX: ignores 'blocking'
+ from threading import _make_thread_handle
+ handle = _make_thread_handle(ident)
+ if done:
+ handle._set_done()
+ return handle
+
+def _create_lock(locked, *args): #XXX: ignores 'blocking'
+ from threading import Lock
+ lock = Lock()
+ if locked:
+ if not lock.acquire(False):
+ raise UnpicklingError("Cannot acquire lock")
+ return lock
+
+def _create_rlock(count, owner, *args): #XXX: ignores 'blocking'
+ lock = RLockType()
+ if owner is not None:
+ lock._acquire_restore((count, owner))
+ if owner and not lock._is_owned():
+ raise UnpicklingError("Cannot acquire lock")
+ return lock
+
+# thanks to matsjoyce for adding all the different file modes
+def _create_filehandle(name, mode, position, closed, open, strictio, fmode, fdata): # buffering=0
+ # only pickles the handle, not the file contents... good? or StringIO(data)?
+ # (for file contents see: http://effbot.org/librarybook/copy-reg.htm)
+ # NOTE: handle special cases first (are there more special cases?)
+ names = {'':sys.__stdin__, '':sys.__stdout__,
+ '':sys.__stderr__} #XXX: better fileno=(0,1,2) ?
+ if name in list(names.keys()):
+ f = names[name] #XXX: safer "f=sys.stdin"
+ elif name == '':
+ f = os.tmpfile()
+ elif name == '':
+ import tempfile
+ f = tempfile.TemporaryFile(mode)
+ else:
+ try:
+ exists = os.path.exists(name)
+ except Exception:
+ exists = False
+ if not exists:
+ if strictio:
+ raise FileNotFoundError("[Errno 2] No such file or directory: '%s'" % name)
+ elif "r" in mode and fmode != FILE_FMODE:
+ name = '' # or os.devnull?
+ current_size = 0 # or maintain position?
+ else:
+ current_size = os.path.getsize(name)
+
+ if position > current_size:
+ if strictio:
+ raise ValueError("invalid buffer size")
+ elif fmode == CONTENTS_FMODE:
+ position = current_size
+ # try to open the file by name
+ # NOTE: has different fileno
+ try:
+ #FIXME: missing: *buffering*, encoding, softspace
+ if fmode == FILE_FMODE:
+ f = open(name, mode if "w" in mode else "w")
+ f.write(fdata)
+ if "w" not in mode:
+ f.close()
+ f = open(name, mode)
+ elif name == '': # file did not exist
+ import tempfile
+ f = tempfile.TemporaryFile(mode)
+ # treat x mode as w mode
+ elif fmode == CONTENTS_FMODE \
+ and ("w" in mode or "x" in mode):
+ # stop truncation when opening
+ flags = os.O_CREAT
+ if "+" in mode:
+ flags |= os.O_RDWR
+ else:
+ flags |= os.O_WRONLY
+ f = os.fdopen(os.open(name, flags), mode)
+ # set name to the correct value
+ r = getattr(f, "buffer", f)
+ r = getattr(r, "raw", r)
+ r.name = name
+ assert f.name == name
+ else:
+ f = open(name, mode)
+ except (IOError, FileNotFoundError):
+ err = sys.exc_info()[1]
+ raise UnpicklingError(err)
+ if closed:
+ f.close()
+ elif position >= 0 and fmode != HANDLE_FMODE:
+ f.seek(position)
+ return f
+
+def _create_stringi(value, position, closed):
+ f = StringIO(value)
+ if closed: f.close()
+ else: f.seek(position)
+ return f
+
+def _create_stringo(value, position, closed):
+ f = StringIO()
+ if closed: f.close()
+ else:
+ f.write(value)
+ f.seek(position)
+ return f
+
+class _itemgetter_helper(object):
+ def __init__(self):
+ self.items = []
+ def __getitem__(self, item):
+ self.items.append(item)
+ return
+
+class _attrgetter_helper(object):
+ def __init__(self, attrs, index=None):
+ self.attrs = attrs
+ self.index = index
+ def __getattribute__(self, attr):
+ attrs = object.__getattribute__(self, "attrs")
+ index = object.__getattribute__(self, "index")
+ if index is None:
+ index = len(attrs)
+ attrs.append(attr)
+ else:
+ attrs[index] = ".".join([attrs[index], attr])
+ return type(self)(attrs, index)
+
+class _dictproxy_helper(dict):
+ def __ror__(self, a):
+ return a
+
+_dictproxy_helper_instance = _dictproxy_helper()
+
+__d = {}
+try:
+ # In CPython 3.9 and later, this trick can be used to exploit the
+ # implementation of the __or__ function of MappingProxyType to get the true
+ # mapping referenced by the proxy. It may work for other implementations,
+ # but is not guaranteed.
+ MAPPING_PROXY_TRICK = __d is (DictProxyType(__d) | _dictproxy_helper_instance)
+except Exception:
+ MAPPING_PROXY_TRICK = False
+del __d
+
+# _CELL_REF and _CELL_EMPTY are used to stay compatible with versions of dill
+# whose _create_cell functions do not have a default value.
+# _CELL_REF can be safely removed entirely (replaced by empty tuples for calls
+# to _create_cell) once breaking changes are allowed.
+_CELL_REF = None
+_CELL_EMPTY = Sentinel('_CELL_EMPTY')
+
+def _create_cell(contents=None):
+ if contents is not _CELL_EMPTY:
+ value = contents
+ return (lambda: value).__closure__[0]
+
+def _create_weakref(obj, *args):
+ from weakref import ref
+ if obj is None: # it's dead
+ from collections import UserDict
+ return ref(UserDict(), *args)
+ return ref(obj, *args)
+
+def _create_weakproxy(obj, callable=False, *args):
+ from weakref import proxy
+ if obj is None: # it's dead
+ if callable: return proxy(lambda x:x, *args)
+ from collections import UserDict
+ return proxy(UserDict(), *args)
+ return proxy(obj, *args)
+
+def _eval_repr(repr_str):
+ return eval(repr_str)
+
+def _create_array(f, args, state, npdict=None):
+ #array = numpy.core.multiarray._reconstruct(*args)
+ array = f(*args)
+ array.__setstate__(state)
+ if npdict is not None: # we also have saved state in __dict__
+ array.__dict__.update(npdict)
+ return array
+
+def _create_dtypemeta(scalar_type):
+ if NumpyDType is True: __hook__() # a bit hacky I think
+ if scalar_type is None:
+ return NumpyDType
+ return type(NumpyDType(scalar_type))
+
+def _create_namedtuple(name, fieldnames, modulename, defaults=None):
+ class_ = _import_module(modulename + '.' + name, safe=True)
+ if class_ is not None:
+ return class_
+ import collections
+ t = collections.namedtuple(name, fieldnames, defaults=defaults, module=modulename)
+ return t
+
+def _create_capsule(pointer, name, context, destructor):
+ attr_found = False
+ try:
+ # based on https://github.com/python/cpython/blob/f4095e53ab708d95e019c909d5928502775ba68f/Objects/capsule.c#L209-L231
+ uname = name.decode('utf8')
+ for i in range(1, uname.count('.')+1):
+ names = uname.rsplit('.', i)
+ try:
+ module = __import__(names[0])
+ except ImportError:
+ pass
+ obj = module
+ for attr in names[1:]:
+ obj = getattr(obj, attr)
+ capsule = obj
+ attr_found = True
+ break
+ except Exception:
+ pass
+
+ if attr_found:
+ if _PyCapsule_IsValid(capsule, name):
+ return capsule
+ raise UnpicklingError("%s object exists at %s but a PyCapsule object was expected." % (type(capsule), name))
+ else:
+ #warnings.warn('Creating a new PyCapsule %s for a C data structure that may not be present in memory. Segmentation faults or other memory errors are possible.' % (name,), UnpicklingWarning)
+ capsule = _PyCapsule_New(pointer, name, destructor)
+ _PyCapsule_SetContext(capsule, context)
+ return capsule
+
+def _getattr(objclass, name, repr_str):
+ # hack to grab the reference directly
+ try: #XXX: works only for __builtin__ ?
+ attr = repr_str.split("'")[3]
+ return eval(attr+'.__dict__["'+name+'"]')
+ except Exception:
+ try:
+ attr = objclass.__dict__
+ if type(attr) is DictProxyType:
+ if sys.hexversion > 0x30f00a0 and name in ('__weakref__','__dict__'):
+ attr = _dictproxy_helper.__dict__[name]
+ else:
+ attr = attr[name]
+ else:
+ attr = getattr(objclass,name)
+ except (AttributeError, KeyError):
+ attr = getattr(objclass,name)
+ return attr
+
+def _get_attr(self, name):
+ # stop recursive pickling
+ return getattr(self, name, None) or getattr(__builtin__, name)
+
+def _import_module(import_name, safe=False):
+ try:
+ if import_name.startswith('__runtime__.'):
+ return sys.modules[import_name]
+ elif '.' in import_name:
+ items = import_name.split('.')
+ module = '.'.join(items[:-1])
+ obj = items[-1]
+ submodule = getattr(__import__(module, None, None, [obj]), obj)
+ if isinstance(submodule, (ModuleType, type)):
+ return submodule
+ return __import__(import_name, None, None, [obj])
+ else:
+ return __import__(import_name)
+ except (ImportError, AttributeError, KeyError):
+ if safe:
+ return None
+ raise
+
+# https://github.com/python/cpython/blob/a8912a0f8d9eba6d502c37d522221f9933e976db/Lib/pickle.py#L322-L333
+def _getattribute(obj, name):
+ for subpath in name.split('.'):
+ if subpath == '':
+ raise AttributeError("Can't get local attribute {!r} on {!r}"
+ .format(name, obj))
+ try:
+ parent = obj
+ obj = getattr(obj, subpath)
+ except AttributeError:
+ raise AttributeError("Can't get attribute {!r} on {!r}"
+ .format(name, obj))
+ return obj, parent
+
+def _locate_function(obj, pickler=None):
+ module_name = getattr(obj, '__module__', None)
+ if module_name in ['__main__', None] or \
+ pickler and is_dill(pickler, child=False) and pickler._session and module_name == pickler._main.__name__:
+ return False
+ if hasattr(obj, '__qualname__'):
+ module = _import_module(module_name, safe=True)
+ try:
+ found, _ = _getattribute(module, obj.__qualname__)
+ return found is obj
+ except AttributeError:
+ return False
+ else:
+ found = _import_module(module_name + '.' + obj.__name__, safe=True)
+ return found is obj
+
+
+def _setitems(dest, source):
+ for k, v in source.items():
+ dest[k] = v
+
+
+def _save_with_postproc(pickler, reduction, is_pickler_dill=None, obj=Getattr.NO_DEFAULT, postproc_list=None):
+ if obj is Getattr.NO_DEFAULT:
+ obj = Reduce(reduction) # pragma: no cover
+
+ if is_pickler_dill is None:
+ is_pickler_dill = is_dill(pickler, child=True)
+ if is_pickler_dill:
+ # assert id(obj) not in pickler._postproc, str(obj) + ' already pushed on stack!'
+ # if not hasattr(pickler, 'x'): pickler.x = 0
+ # print(pickler.x*' ', 'push', obj, id(obj), pickler._recurse)
+ # pickler.x += 1
+ if postproc_list is None:
+ postproc_list = []
+
+ # Recursive object not supported. Default to a global instead.
+ if id(obj) in pickler._postproc:
+ name = '%s.%s ' % (obj.__module__, getattr(obj, '__qualname__', obj.__name__)) if hasattr(obj, '__module__') else ''
+ warnings.warn('Cannot pickle %r: %shas recursive self-references that trigger a RecursionError.' % (obj, name), PicklingWarning)
+ pickler.save_global(obj)
+ return
+ pickler._postproc[id(obj)] = postproc_list
+
+ # TODO: Use state_setter in Python 3.8 to allow for faster cPickle implementations
+ pickler.save_reduce(*reduction, obj=obj)
+
+ if is_pickler_dill:
+ # pickler.x -= 1
+ # print(pickler.x*' ', 'pop', obj, id(obj))
+ postproc = pickler._postproc.pop(id(obj))
+ # assert postproc_list == postproc, 'Stack tampered!'
+ for reduction in reversed(postproc):
+ if reduction[0] is _setitems:
+ # use the internal machinery of pickle.py to speedup when
+ # updating a dictionary in postproc
+ dest, source = reduction[1]
+ if source:
+ pickler.write(pickler.get(pickler.memo[id(dest)][0]))
+ if sys.hexversion < 0x30e00a1:
+ pickler._batch_setitems(iter(source.items()))
+ else:
+ pickler._batch_setitems(iter(source.items()), obj=obj)
+ else:
+ # Updating with an empty dictionary. Same as doing nothing.
+ continue
+ else:
+ pickler.save_reduce(*reduction)
+ # pop None created by calling preprocessing step off stack
+ pickler.write(POP)
+
+#@register(CodeType)
+#def save_code(pickler, obj):
+# logger.trace(pickler, "Co: %s", obj)
+# pickler.save_reduce(_unmarshal, (marshal.dumps(obj),), obj=obj)
+# logger.trace(pickler, "# Co")
+# return
+
+# The following function is based on 'save_codeobject' from 'cloudpickle'
+# Copyright (c) 2012, Regents of the University of California.
+# Copyright (c) 2009 `PiCloud, Inc. `_.
+# License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE
+@register(CodeType)
+def save_code(pickler, obj):
+ logger.trace(pickler, "Co: %s", obj)
+ if hasattr(obj, "co_endlinetable"): # python 3.11a (20 args)
+ args = (
+ obj.co_lnotab, # for < python 3.10 [not counted in args]
+ obj.co_argcount, obj.co_posonlyargcount,
+ obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
+ obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
+ obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname,
+ obj.co_firstlineno, obj.co_linetable, obj.co_endlinetable,
+ obj.co_columntable, obj.co_exceptiontable, obj.co_freevars,
+ obj.co_cellvars
+ )
+ elif hasattr(obj, "co_exceptiontable"): # python 3.11 (18 args)
+ with warnings.catch_warnings():
+ if not OLD312a7: # issue 597
+ warnings.filterwarnings('ignore', category=DeprecationWarning)
+ args = (
+ obj.co_lnotab, # for < python 3.10 [not counted in args]
+ obj.co_argcount, obj.co_posonlyargcount,
+ obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
+ obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
+ obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname,
+ obj.co_firstlineno, obj.co_linetable, obj.co_exceptiontable,
+ obj.co_freevars, obj.co_cellvars
+ )
+ elif hasattr(obj, "co_qualname"): # pypy 3.11 7.3.19+ (17 args)
+ args = (
+ obj.co_lnotab, obj.co_argcount, obj.co_posonlyargcount,
+ obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
+ obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
+ obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname,
+ obj.co_firstlineno, obj.co_linetable, obj.co_freevars,
+ obj.co_cellvars
+ )
+ elif hasattr(obj, "co_linetable"): # python 3.10 (16 args)
+ args = (
+ obj.co_lnotab, # for < python 3.10 [not counted in args]
+ obj.co_argcount, obj.co_posonlyargcount,
+ obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
+ obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
+ obj.co_varnames, obj.co_filename, obj.co_name,
+ obj.co_firstlineno, obj.co_linetable, obj.co_freevars,
+ obj.co_cellvars
+ )
+ elif hasattr(obj, "co_posonlyargcount"): # python 3.8 (16 args)
+ args = (
+ obj.co_argcount, obj.co_posonlyargcount,
+ obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
+ obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
+ obj.co_varnames, obj.co_filename, obj.co_name,
+ obj.co_firstlineno, obj.co_lnotab, obj.co_freevars,
+ obj.co_cellvars
+ )
+ else: # python 3.7 (15 args)
+ args = (
+ obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals,
+ obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts,
+ obj.co_names, obj.co_varnames, obj.co_filename,
+ obj.co_name, obj.co_firstlineno, obj.co_lnotab,
+ obj.co_freevars, obj.co_cellvars
+ )
+
+ pickler.save_reduce(_create_code, args, obj=obj)
+ logger.trace(pickler, "# Co")
+ return
+
+def _repr_dict(obj):
+ """Make a short string representation of a dictionary."""
+ return "<%s object at %#012x>" % (type(obj).__name__, id(obj))
+
+@register(dict)
+def save_module_dict(pickler, obj):
+ if is_dill(pickler, child=False) and obj == pickler._main.__dict__ and \
+ not (pickler._session and pickler._first_pass):
+ logger.trace(pickler, "D1: %s", _repr_dict(obj)) # obj
+ pickler.write(bytes('c__builtin__\n__main__\n', 'UTF-8'))
+ logger.trace(pickler, "# D1")
+ elif (not is_dill(pickler, child=False)) and (obj == _main_module.__dict__):
+ logger.trace(pickler, "D3: %s", _repr_dict(obj)) # obj
+ pickler.write(bytes('c__main__\n__dict__\n', 'UTF-8')) #XXX: works in general?
+ logger.trace(pickler, "# D3")
+ elif '__name__' in obj and obj != _main_module.__dict__ \
+ and type(obj['__name__']) is str \
+ and obj is getattr(_import_module(obj['__name__'],True), '__dict__', None):
+ logger.trace(pickler, "D4: %s", _repr_dict(obj)) # obj
+ pickler.write(bytes('c%s\n__dict__\n' % obj['__name__'], 'UTF-8'))
+ logger.trace(pickler, "# D4")
+ else:
+ logger.trace(pickler, "D2: %s", _repr_dict(obj)) # obj
+ if is_dill(pickler, child=False) and pickler._session:
+ # we only care about session the first pass thru
+ pickler._first_pass = False
+ StockPickler.save_dict(pickler, obj)
+ logger.trace(pickler, "# D2")
+ return
+
+
+if not OLD310 and MAPPING_PROXY_TRICK:
+ def save_dict_view(dicttype):
+ def save_dict_view_for_function(func):
+ def _save_dict_view(pickler, obj):
+ logger.trace(pickler, "Dkvi: <%s>", obj)
+ mapping = obj.mapping | _dictproxy_helper_instance
+ pickler.save_reduce(func, (mapping,), obj=obj)
+ logger.trace(pickler, "# Dkvi")
+ return _save_dict_view
+ return [
+ (funcname, save_dict_view_for_function(getattr(dicttype, funcname)))
+ for funcname in ('keys', 'values', 'items')
+ ]
+else:
+ # The following functions are based on 'cloudpickle'
+ # https://github.com/cloudpipe/cloudpickle/blob/5d89947288a18029672596a4d719093cc6d5a412/cloudpickle/cloudpickle.py#L922-L940
+ # Copyright (c) 2012, Regents of the University of California.
+ # Copyright (c) 2009 `PiCloud, Inc. `_.
+ # License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE
+ def save_dict_view(dicttype):
+ def save_dict_keys(pickler, obj):
+ logger.trace(pickler, "Dk: <%s>", obj)
+ dict_constructor = _shims.Reduce(dicttype.fromkeys, (list(obj),))
+ pickler.save_reduce(dicttype.keys, (dict_constructor,), obj=obj)
+ logger.trace(pickler, "# Dk")
+
+ def save_dict_values(pickler, obj):
+ logger.trace(pickler, "Dv: <%s>", obj)
+ dict_constructor = _shims.Reduce(dicttype, (enumerate(obj),))
+ pickler.save_reduce(dicttype.values, (dict_constructor,), obj=obj)
+ logger.trace(pickler, "# Dv")
+
+ def save_dict_items(pickler, obj):
+ logger.trace(pickler, "Di: <%s>", obj)
+ pickler.save_reduce(dicttype.items, (dicttype(obj),), obj=obj)
+ logger.trace(pickler, "# Di")
+
+ return (
+ ('keys', save_dict_keys),
+ ('values', save_dict_values),
+ ('items', save_dict_items)
+ )
+
+for __dicttype in (
+ dict,
+ OrderedDict
+):
+ __obj = __dicttype()
+ for __funcname, __savefunc in save_dict_view(__dicttype):
+ __tview = type(getattr(__obj, __funcname)())
+ if __tview not in Pickler.dispatch:
+ Pickler.dispatch[__tview] = __savefunc
+del __dicttype, __obj, __funcname, __tview, __savefunc
+
+
+@register(ClassType)
+def save_classobj(pickler, obj): #FIXME: enable pickler._byref
+ if not _locate_function(obj, pickler):
+ logger.trace(pickler, "C1: %s", obj)
+ pickler.save_reduce(ClassType, (obj.__name__, obj.__bases__,
+ obj.__dict__), obj=obj)
+ #XXX: or obj.__dict__.copy()), obj=obj) ?
+ logger.trace(pickler, "# C1")
+ else:
+ logger.trace(pickler, "C2: %s", obj)
+ name = getattr(obj, '__qualname__', getattr(obj, '__name__', None))
+ StockPickler.save_global(pickler, obj, name=name)
+ logger.trace(pickler, "# C2")
+ return
+
+@register(typing._GenericAlias)
+def save_generic_alias(pickler, obj):
+ args = obj.__args__
+ if type(obj.__reduce__()) is str:
+ logger.trace(pickler, "Ga0: %s", obj)
+ StockPickler.save_global(pickler, obj, name=obj.__reduce__())
+ logger.trace(pickler, "# Ga0")
+ elif obj.__origin__ is tuple and (not args or args == ((),)):
+ logger.trace(pickler, "Ga1: %s", obj)
+ pickler.save_reduce(_create_typing_tuple, (args,), obj=obj)
+ logger.trace(pickler, "# Ga1")
+ else:
+ logger.trace(pickler, "Ga2: %s", obj)
+ StockPickler.save_reduce(pickler, *obj.__reduce__(), obj=obj)
+ logger.trace(pickler, "# Ga2")
+ return
+
+if ThreadHandleType:
+ @register(ThreadHandleType)
+ def save_thread_handle(pickler, obj):
+ logger.trace(pickler, "Th: %s", obj)
+ pickler.save_reduce(_create_thread_handle, (obj.ident, obj.is_done()), obj=obj)
+ logger.trace(pickler, "# Th")
+ return
+
+@register(LockType) #XXX: copied Thread will have new Event (due to new Lock)
+def save_lock(pickler, obj):
+ logger.trace(pickler, "Lo: %s", obj)
+ pickler.save_reduce(_create_lock, (obj.locked(),), obj=obj)
+ logger.trace(pickler, "# Lo")
+ return
+
+@register(RLockType)
+def save_rlock(pickler, obj):
+ logger.trace(pickler, "RL: %s", obj)
+ r = obj.__repr__() # don't use _release_save as it unlocks the lock
+ count = int(r.split('count=')[1].split()[0].rstrip('>'))
+ owner = int(r.split('owner=')[1].split()[0])
+ pickler.save_reduce(_create_rlock, (count,owner,), obj=obj)
+ logger.trace(pickler, "# RL")
+ return
+
+#@register(SocketType) #FIXME: causes multiprocess test_pickling FAIL
+def save_socket(pickler, obj):
+ logger.trace(pickler, "So: %s", obj)
+ pickler.save_reduce(*reduce_socket(obj))
+ logger.trace(pickler, "# So")
+ return
+
+def _save_file(pickler, obj, open_):
+ if obj.closed:
+ position = 0
+ else:
+ obj.flush()
+ if obj in (sys.__stdout__, sys.__stderr__, sys.__stdin__):
+ position = -1
+ else:
+ position = obj.tell()
+ if is_dill(pickler, child=True) and pickler._fmode == FILE_FMODE:
+ f = open_(obj.name, "r")
+ fdata = f.read()
+ f.close()
+ else:
+ fdata = ""
+ if is_dill(pickler, child=True):
+ strictio = pickler._strictio
+ fmode = pickler._fmode
+ else:
+ strictio = False
+ fmode = 0 # HANDLE_FMODE
+ pickler.save_reduce(_create_filehandle, (obj.name, obj.mode, position,
+ obj.closed, open_, strictio,
+ fmode, fdata), obj=obj)
+ return
+
+
+@register(FileType) #XXX: in 3.x has buffer=0, needs different _create?
+@register(BufferedReaderType)
+@register(BufferedWriterType)
+@register(TextWrapperType)
+def save_file(pickler, obj):
+ logger.trace(pickler, "Fi: %s", obj)
+ f = _save_file(pickler, obj, open)
+ logger.trace(pickler, "# Fi")
+ return f
+
+if BufferedRandomType:
+ @register(BufferedRandomType)
+ def save_file(pickler, obj):
+ logger.trace(pickler, "Fi: %s", obj)
+ f = _save_file(pickler, obj, open)
+ logger.trace(pickler, "# Fi")
+ return f
+
+if PyTextWrapperType:
+ @register(PyBufferedReaderType)
+ @register(PyBufferedWriterType)
+ @register(PyTextWrapperType)
+ def save_file(pickler, obj):
+ logger.trace(pickler, "Fi: %s", obj)
+ f = _save_file(pickler, obj, _open)
+ logger.trace(pickler, "# Fi")
+ return f
+
+ if PyBufferedRandomType:
+ @register(PyBufferedRandomType)
+ def save_file(pickler, obj):
+ logger.trace(pickler, "Fi: %s", obj)
+ f = _save_file(pickler, obj, _open)
+ logger.trace(pickler, "# Fi")
+ return f
+
+
+# The following two functions are based on 'saveCStringIoInput'
+# and 'saveCStringIoOutput' from spickle
+# Copyright (c) 2011 by science+computing ag
+# License: http://www.apache.org/licenses/LICENSE-2.0
+if InputType:
+ @register(InputType)
+ def save_stringi(pickler, obj):
+ logger.trace(pickler, "Io: %s", obj)
+ if obj.closed:
+ value = ''; position = 0
+ else:
+ value = obj.getvalue(); position = obj.tell()
+ pickler.save_reduce(_create_stringi, (value, position, \
+ obj.closed), obj=obj)
+ logger.trace(pickler, "# Io")
+ return
+
+ @register(OutputType)
+ def save_stringo(pickler, obj):
+ logger.trace(pickler, "Io: %s", obj)
+ if obj.closed:
+ value = ''; position = 0
+ else:
+ value = obj.getvalue(); position = obj.tell()
+ pickler.save_reduce(_create_stringo, (value, position, \
+ obj.closed), obj=obj)
+ logger.trace(pickler, "# Io")
+ return
+
+if LRUCacheType is not None:
+ from functools import lru_cache
+ @register(LRUCacheType)
+ def save_lru_cache(pickler, obj):
+ logger.trace(pickler, "LRU: %s", obj)
+ if OLD39:
+ kwargs = obj.cache_info()
+ args = (kwargs.maxsize,)
+ else:
+ kwargs = obj.cache_parameters()
+ args = (kwargs['maxsize'], kwargs['typed'])
+ if args != lru_cache.__defaults__:
+ wrapper = Reduce(lru_cache, args, is_callable=True)
+ else:
+ wrapper = lru_cache
+ pickler.save_reduce(wrapper, (obj.__wrapped__,), obj=obj)
+ logger.trace(pickler, "# LRU")
+ return
+
+@register(SuperType)
+def save_super(pickler, obj):
+ logger.trace(pickler, "Su: %s", obj)
+ pickler.save_reduce(super, (obj.__thisclass__, obj.__self__), obj=obj)
+ logger.trace(pickler, "# Su")
+ return
+
+if IS_PYPY:
+ @register(MethodType)
+ def save_instancemethod0(pickler, obj):
+ code = getattr(obj.__func__, '__code__', None)
+ if code is not None and type(code) is not CodeType \
+ and getattr(obj.__self__, obj.__name__) == obj:
+ # Some PyPy builtin functions have no module name
+ logger.trace(pickler, "Me2: %s", obj)
+ # TODO: verify that this works for all PyPy builtin methods
+ pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj)
+ logger.trace(pickler, "# Me2")
+ return
+
+ logger.trace(pickler, "Me1: %s", obj)
+ pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj)
+ logger.trace(pickler, "# Me1")
+ return
+else:
+ @register(MethodType)
+ def save_instancemethod0(pickler, obj):
+ logger.trace(pickler, "Me1: %s", obj)
+ pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj)
+ logger.trace(pickler, "# Me1")
+ return
+
+if not IS_PYPY:
+ @register(MemberDescriptorType)
+ @register(GetSetDescriptorType)
+ @register(MethodDescriptorType)
+ @register(WrapperDescriptorType)
+ @register(ClassMethodDescriptorType)
+ def save_wrapper_descriptor(pickler, obj):
+ logger.trace(pickler, "Wr: %s", obj)
+ pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__,
+ obj.__repr__()), obj=obj)
+ logger.trace(pickler, "# Wr")
+ return
+else:
+ @register(MemberDescriptorType)
+ @register(GetSetDescriptorType)
+ def save_wrapper_descriptor(pickler, obj):
+ logger.trace(pickler, "Wr: %s", obj)
+ pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__,
+ obj.__repr__()), obj=obj)
+ logger.trace(pickler, "# Wr")
+ return
+
+@register(CellType)
+def save_cell(pickler, obj):
+ try:
+ f = obj.cell_contents
+ except ValueError: # cell is empty
+ logger.trace(pickler, "Ce3: %s", obj)
+ # _shims._CELL_EMPTY is defined in _shims.py to support PyPy 2.7.
+ # It unpickles to a sentinel object _dill._CELL_EMPTY, also created in
+ # _shims.py. This object is not present in Python 3 because the cell's
+ # contents can be deleted in newer versions of Python. The reduce object
+ # will instead unpickle to None if unpickled in Python 3.
+
+ # When breaking changes are made to dill, (_shims._CELL_EMPTY,) can
+ # be replaced by () OR the delattr function can be removed repending on
+ # whichever is more convienient.
+ pickler.save_reduce(_create_cell, (_shims._CELL_EMPTY,), obj=obj)
+ # Call the function _delattr on the cell's cell_contents attribute
+ # The result of this function call will be None
+ pickler.save_reduce(_shims._delattr, (obj, 'cell_contents'))
+ # pop None created by calling _delattr off stack
+ pickler.write(POP)
+ logger.trace(pickler, "# Ce3")
+ return
+ if is_dill(pickler, child=True):
+ if id(f) in pickler._postproc:
+ # Already seen. Add to its postprocessing.
+ postproc = pickler._postproc[id(f)]
+ else:
+ # Haven't seen it. Add to the highest possible object and set its
+ # value as late as possible to prevent cycle.
+ postproc = next(iter(pickler._postproc.values()), None)
+ if postproc is not None:
+ logger.trace(pickler, "Ce2: %s", obj)
+ # _CELL_REF is defined in _shims.py to support older versions of
+ # dill. When breaking changes are made to dill, (_CELL_REF,) can
+ # be replaced by ()
+ pickler.save_reduce(_create_cell, (_CELL_REF,), obj=obj)
+ postproc.append((_shims._setattr, (obj, 'cell_contents', f)))
+ logger.trace(pickler, "# Ce2")
+ return
+ logger.trace(pickler, "Ce1: %s", obj)
+ pickler.save_reduce(_create_cell, (f,), obj=obj)
+ logger.trace(pickler, "# Ce1")
+ return
+
+if MAPPING_PROXY_TRICK:
+ @register(DictProxyType)
+ def save_dictproxy(pickler, obj):
+ logger.trace(pickler, "Mp: %s", _repr_dict(obj)) # obj
+ mapping = obj | _dictproxy_helper_instance
+ pickler.save_reduce(DictProxyType, (mapping,), obj=obj)
+ logger.trace(pickler, "# Mp")
+ return
+else:
+ @register(DictProxyType)
+ def save_dictproxy(pickler, obj):
+ logger.trace(pickler, "Mp: %s", _repr_dict(obj)) # obj
+ pickler.save_reduce(DictProxyType, (obj.copy(),), obj=obj)
+ logger.trace(pickler, "# Mp")
+ return
+
+@register(SliceType)
+def save_slice(pickler, obj):
+ logger.trace(pickler, "Sl: %s", obj)
+ pickler.save_reduce(slice, (obj.start, obj.stop, obj.step), obj=obj)
+ logger.trace(pickler, "# Sl")
+ return
+
+@register(XRangeType)
+@register(EllipsisType)
+@register(NotImplementedType)
+def save_singleton(pickler, obj):
+ logger.trace(pickler, "Si: %s", obj)
+ pickler.save_reduce(_eval_repr, (obj.__repr__(),), obj=obj)
+ logger.trace(pickler, "# Si")
+ return
+
+def _proxy_helper(obj): # a dead proxy returns a reference to None
+ """get memory address of proxy's reference object"""
+ _repr = repr(obj)
+ try: _str = str(obj)
+ except ReferenceError: # it's a dead proxy
+ return id(None)
+ if _str == _repr: return id(obj) # it's a repr
+ try: # either way, it's a proxy from here
+ address = int(_str.rstrip('>').split(' at ')[-1], base=16)
+ except ValueError: # special case: proxy of a 'type'
+ if not IS_PYPY:
+ address = int(_repr.rstrip('>').split(' at ')[-1], base=16)
+ else:
+ objects = iter(gc.get_objects())
+ for _obj in objects:
+ if repr(_obj) == _str: return id(_obj)
+ # all bad below... nothing found so throw ReferenceError
+ msg = "Cannot reference object for proxy at '%s'" % id(obj)
+ raise ReferenceError(msg)
+ return address
+
+def _locate_object(address, module=None):
+ """get object located at the given memory address (inverse of id(obj))"""
+ special = [None, True, False] #XXX: more...?
+ for obj in special:
+ if address == id(obj): return obj
+ if module:
+ objects = iter(module.__dict__.values())
+ else: objects = iter(gc.get_objects())
+ for obj in objects:
+ if address == id(obj): return obj
+ # all bad below... nothing found so throw ReferenceError or TypeError
+ try: address = hex(address)
+ except TypeError:
+ raise TypeError("'%s' is not a valid memory address" % str(address))
+ raise ReferenceError("Cannot reference object at '%s'" % address)
+
+@register(ReferenceType)
+def save_weakref(pickler, obj):
+ refobj = obj()
+ logger.trace(pickler, "R1: %s", obj)
+ #refobj = ctypes.pythonapi.PyWeakref_GetObject(obj) # dead returns "None"
+ pickler.save_reduce(_create_weakref, (refobj,), obj=obj)
+ logger.trace(pickler, "# R1")
+ return
+
+@register(ProxyType)
+@register(CallableProxyType)
+def save_weakproxy(pickler, obj):
+ # Must do string substitution here and use %r to avoid ReferenceError.
+ logger.trace(pickler, "R2: %r" % obj)
+ refobj = _locate_object(_proxy_helper(obj))
+ pickler.save_reduce(_create_weakproxy, (refobj, callable(obj)), obj=obj)
+ logger.trace(pickler, "# R2")
+ return
+
+def _is_builtin_module(module):
+ if not hasattr(module, "__file__"): return True
+ if module.__file__ is None: return False
+ # If a module file name starts with prefix, it should be a builtin
+ # module, so should always be pickled as a reference.
+ names = ["base_prefix", "base_exec_prefix", "exec_prefix", "prefix", "real_prefix"]
+ rp = os.path.realpath
+ # See https://github.com/uqfoundation/dill/issues/566
+ return (
+ any(
+ module.__file__.startswith(getattr(sys, name))
+ or rp(module.__file__).startswith(rp(getattr(sys, name)))
+ for name in names
+ if hasattr(sys, name)
+ )
+ or module.__file__.endswith(EXTENSION_SUFFIXES)
+ or 'site-packages' in module.__file__
+ )
+
+def _is_imported_module(module):
+ return getattr(module, '__loader__', None) is not None or module in sys.modules.values()
+
+@register(ModuleType)
+def save_module(pickler, obj):
+ if False: #_use_diff:
+ if obj.__name__.split('.', 1)[0] != "dill":
+ try:
+ changed = diff.whats_changed(obj, seen=pickler._diff_cache)[0]
+ except RuntimeError: # not memorised module, probably part of dill
+ pass
+ else:
+ logger.trace(pickler, "M2: %s with diff", obj)
+ logger.info("Diff: %s", changed.keys())
+ pickler.save_reduce(_import_module, (obj.__name__,), obj=obj,
+ state=changed)
+ logger.trace(pickler, "# M2")
+ return
+
+ logger.trace(pickler, "M1: %s", obj)
+ pickler.save_reduce(_import_module, (obj.__name__,), obj=obj)
+ logger.trace(pickler, "# M1")
+ else:
+ builtin_mod = _is_builtin_module(obj)
+ is_session_main = is_dill(pickler, child=True) and obj is pickler._main
+ if (obj.__name__ not in ("builtins", "dill", "dill._dill") and not builtin_mod
+ or is_session_main):
+ logger.trace(pickler, "M1: %s", obj)
+ # Hack for handling module-type objects in load_module().
+ mod_name = obj.__name__ if _is_imported_module(obj) else '__runtime__.%s' % obj.__name__
+ # Second references are saved as __builtin__.__main__ in save_module_dict().
+ main_dict = obj.__dict__.copy()
+ for item in ('__builtins__', '__loader__'):
+ main_dict.pop(item, None)
+ for item in IPYTHON_SINGLETONS: #pragma: no cover
+ if getattr(main_dict.get(item), '__module__', '').startswith('IPython'):
+ del main_dict[item]
+ pickler.save_reduce(_import_module, (mod_name,), obj=obj, state=main_dict)
+ logger.trace(pickler, "# M1")
+ elif obj.__name__ == "dill._dill":
+ logger.trace(pickler, "M2: %s", obj)
+ pickler.save_global(obj, name="_dill")
+ logger.trace(pickler, "# M2")
+ else:
+ logger.trace(pickler, "M2: %s", obj)
+ pickler.save_reduce(_import_module, (obj.__name__,), obj=obj)
+ logger.trace(pickler, "# M2")
+ return
+
+# The following function is based on '_extract_class_dict' from 'cloudpickle'
+# Copyright (c) 2012, Regents of the University of California.
+# Copyright (c) 2009 `PiCloud, Inc. `_.
+# License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE
+def _get_typedict_type(cls, clsdict, attrs, postproc_list):
+ """Retrieve a copy of the dict of a class without the inherited methods"""
+ if len(cls.__bases__) == 1:
+ inherited_dict = cls.__bases__[0].__dict__
+ else:
+ inherited_dict = {}
+ for base in reversed(cls.__bases__):
+ inherited_dict.update(base.__dict__)
+ to_remove = []
+ for name, value in dict.items(clsdict):
+ try:
+ base_value = inherited_dict[name]
+ if value is base_value and hasattr(value, '__qualname__'):
+ to_remove.append(name)
+ except KeyError:
+ pass
+ for name in to_remove:
+ dict.pop(clsdict, name)
+
+ if issubclass(type(cls), type):
+ clsdict.pop('__dict__', None)
+ clsdict.pop('__weakref__', None)
+ # clsdict.pop('__prepare__', None)
+ return clsdict, attrs
+
+def _get_typedict_abc(obj, _dict, attrs, postproc_list):
+ if hasattr(abc, '_get_dump'):
+ (registry, _, _, _) = abc._get_dump(obj)
+ register = obj.register
+ postproc_list.extend((register, (reg(),)) for reg in registry)
+ elif hasattr(obj, '_abc_registry'):
+ registry = obj._abc_registry
+ register = obj.register
+ postproc_list.extend((register, (reg,)) for reg in registry)
+ else:
+ raise PicklingError("Cannot find registry of ABC %s", obj)
+
+ if '_abc_registry' in _dict:
+ _dict.pop('_abc_registry', None)
+ _dict.pop('_abc_cache', None)
+ _dict.pop('_abc_negative_cache', None)
+ # _dict.pop('_abc_negative_cache_version', None)
+ else:
+ _dict.pop('_abc_impl', None)
+ return _dict, attrs
+
+@register(TypeType)
+def save_type(pickler, obj, postproc_list=None):
+ if obj in _typemap:
+ logger.trace(pickler, "T1: %s", obj)
+ # if obj in _incedental_types:
+ # warnings.warn('Type %r may only exist on this implementation of Python and cannot be unpickled in other implementations.' % (obj,), PicklingWarning)
+ pickler.save_reduce(_load_type, (_typemap[obj],), obj=obj)
+ logger.trace(pickler, "# T1")
+ elif obj.__bases__ == (tuple,) and all([hasattr(obj, attr) for attr in ('_fields','_asdict','_make','_replace')]):
+ # special case: namedtuples
+ logger.trace(pickler, "T6: %s", obj)
+
+ obj_name = getattr(obj, '__qualname__', getattr(obj, '__name__', None))
+ if obj.__name__ != obj_name:
+ if postproc_list is None:
+ postproc_list = []
+ postproc_list.append((setattr, (obj, '__qualname__', obj_name)))
+
+ if not obj._field_defaults:
+ _save_with_postproc(pickler, (_create_namedtuple, (obj.__name__, obj._fields, obj.__module__)), obj=obj, postproc_list=postproc_list)
+ else:
+ defaults = [obj._field_defaults[field] for field in obj._fields if field in obj._field_defaults]
+ _save_with_postproc(pickler, (_create_namedtuple, (obj.__name__, obj._fields, obj.__module__, defaults)), obj=obj, postproc_list=postproc_list)
+ logger.trace(pickler, "# T6")
+ return
+
+ # special caes: NoneType, NotImplementedType, EllipsisType, EnumMeta, etc
+ elif obj is type(None):
+ logger.trace(pickler, "T7: %s", obj)
+ #XXX: pickler.save_reduce(type, (None,), obj=obj)
+ pickler.write(GLOBAL + b'__builtin__\nNoneType\n')
+ logger.trace(pickler, "# T7")
+ elif obj is NotImplementedType:
+ logger.trace(pickler, "T7: %s", obj)
+ pickler.save_reduce(type, (NotImplemented,), obj=obj)
+ logger.trace(pickler, "# T7")
+ elif obj is EllipsisType:
+ logger.trace(pickler, "T7: %s", obj)
+ pickler.save_reduce(type, (Ellipsis,), obj=obj)
+ logger.trace(pickler, "# T7")
+ elif obj is EnumMeta:
+ logger.trace(pickler, "T7: %s", obj)
+ pickler.write(GLOBAL + b'enum\nEnumMeta\n')
+ logger.trace(pickler, "# T7")
+ elif obj is ExceptHookArgsType: #NOTE: must be after NoneType for pypy
+ logger.trace(pickler, "T7: %s", obj)
+ pickler.write(GLOBAL + b'threading\nExceptHookArgs\n')
+ logger.trace(pickler, "# T7")
+
+ else:
+ _byref = getattr(pickler, '_byref', None)
+ obj_recursive = id(obj) in getattr(pickler, '_postproc', ())
+ incorrectly_named = not _locate_function(obj, pickler)
+ if not _byref and not obj_recursive and incorrectly_named: # not a function, but the name was held over
+ if postproc_list is None:
+ postproc_list = []
+
+ # thanks to Tom Stepleton pointing out pickler._session unneeded
+ logger.trace(pickler, "T2: %s", obj)
+ _dict, attrs = _get_typedict_type(obj, obj.__dict__.copy(), None, postproc_list) # copy dict proxy to a dict
+
+ #print (_dict)
+ #print ("%s\n%s" % (type(obj), obj.__name__))
+ #print ("%s\n%s" % (obj.__bases__, obj.__dict__))
+ slots = _dict.get('__slots__', ())
+ if type(slots) == str:
+ # __slots__ accepts a single string
+ slots = (slots,)
+
+ for name in slots:
+ _dict.pop(name, None)
+
+ if isinstance(obj, abc.ABCMeta):
+ logger.trace(pickler, "ABC: %s", obj)
+ _dict, attrs = _get_typedict_abc(obj, _dict, attrs, postproc_list)
+ logger.trace(pickler, "# ABC")
+
+ qualname = getattr(obj, '__qualname__', None)
+ if attrs is not None:
+ for k, v in attrs.items():
+ postproc_list.append((setattr, (obj, k, v)))
+ # TODO: Consider using the state argument to save_reduce?
+ if qualname is not None:
+ postproc_list.append((setattr, (obj, '__qualname__', qualname)))
+
+ if not hasattr(obj, '__orig_bases__'):
+ _save_with_postproc(pickler, (_create_type, (
+ type(obj), obj.__name__, obj.__bases__, _dict
+ )), obj=obj, postproc_list=postproc_list)
+ else:
+ # This case will always work, but might be overkill.
+ _metadict = {
+ 'metaclass': type(obj)
+ }
+
+ if _dict:
+ _dict_update = PartialType(_setitems, source=_dict)
+ else:
+ _dict_update = None
+
+ _save_with_postproc(pickler, (new_class, (
+ obj.__name__, obj.__orig_bases__, _metadict, _dict_update
+ )), obj=obj, postproc_list=postproc_list)
+ logger.trace(pickler, "# T2")
+ else:
+ obj_name = getattr(obj, '__qualname__', getattr(obj, '__name__', None))
+ logger.trace(pickler, "T4: %s", obj)
+ if incorrectly_named:
+ warnings.warn(
+ "Cannot locate reference to %r." % (obj,),
+ PicklingWarning,
+ stacklevel=3,
+ )
+ if obj_recursive:
+ warnings.warn(
+ "Cannot pickle %r: %s.%s has recursive self-references that "
+ "trigger a RecursionError." % (obj, obj.__module__, obj_name),
+ PicklingWarning,
+ stacklevel=3,
+ )
+ #print (obj.__dict__)
+ #print ("%s\n%s" % (type(obj), obj.__name__))
+ #print ("%s\n%s" % (obj.__bases__, obj.__dict__))
+ StockPickler.save_global(pickler, obj, name=obj_name)
+ logger.trace(pickler, "# T4")
+ return
+
+@register(property)
+@register(abc.abstractproperty)
+def save_property(pickler, obj):
+ logger.trace(pickler, "Pr: %s", obj)
+ pickler.save_reduce(type(obj), (obj.fget, obj.fset, obj.fdel, obj.__doc__),
+ obj=obj)
+ logger.trace(pickler, "# Pr")
+
+@register(staticmethod)
+@register(classmethod)
+@register(abc.abstractstaticmethod)
+@register(abc.abstractclassmethod)
+def save_classmethod(pickler, obj):
+ logger.trace(pickler, "Cm: %s", obj)
+ orig_func = obj.__func__
+
+ # if type(obj.__dict__) is dict:
+ # if obj.__dict__:
+ # state = obj.__dict__
+ # else:
+ # state = None
+ # else:
+ # state = (None, {'__dict__', obj.__dict__})
+
+ pickler.save_reduce(type(obj), (orig_func,), obj=obj)
+ logger.trace(pickler, "# Cm")
+
+@register(FunctionType)
+def save_function(pickler, obj):
+ if not _locate_function(obj, pickler):
+ if type(obj.__code__) is not CodeType:
+ # Some PyPy builtin functions have no module name, and thus are not
+ # able to be located
+ module_name = getattr(obj, '__module__', None)
+ if module_name is None:
+ module_name = __builtin__.__name__
+ module = _import_module(module_name, safe=True)
+ _pypy_builtin = False
+ try:
+ found, _ = _getattribute(module, obj.__qualname__)
+ if getattr(found, '__func__', None) is obj:
+ _pypy_builtin = True
+ except AttributeError:
+ pass
+
+ if _pypy_builtin:
+ logger.trace(pickler, "F3: %s", obj)
+ pickler.save_reduce(getattr, (found, '__func__'), obj=obj)
+ logger.trace(pickler, "# F3")
+ return
+
+ logger.trace(pickler, "F1: %s", obj)
+ _recurse = getattr(pickler, '_recurse', None)
+ _postproc = getattr(pickler, '_postproc', None)
+ _main_modified = getattr(pickler, '_main_modified', None)
+ _original_main = getattr(pickler, '_original_main', __builtin__)#'None'
+ postproc_list = []
+ if _recurse:
+ # recurse to get all globals referred to by obj
+ from .detect import globalvars
+ globs_copy = globalvars(obj, recurse=True, builtin=True)
+
+ # Add the name of the module to the globs dictionary to prevent
+ # the duplication of the dictionary. Pickle the unpopulated
+ # globals dictionary and set the remaining items after the function
+ # is created to correctly handle recursion.
+ globs = {'__name__': obj.__module__}
+ else:
+ globs_copy = obj.__globals__
+
+ # If the globals is the __dict__ from the module being saved as a
+ # session, substitute it by the dictionary being actually saved.
+ if _main_modified and globs_copy is _original_main.__dict__:
+ globs_copy = getattr(pickler, '_main', _original_main).__dict__
+ globs = globs_copy
+ # If the globals is a module __dict__, do not save it in the pickle.
+ elif globs_copy is not None and obj.__module__ is not None and \
+ getattr(_import_module(obj.__module__, True), '__dict__', None) is globs_copy:
+ globs = globs_copy
+ else:
+ globs = {'__name__': obj.__module__}
+
+ if globs_copy is not None and globs is not globs_copy:
+ # In the case that the globals are copied, we need to ensure that
+ # the globals dictionary is updated when all objects in the
+ # dictionary are already created.
+ glob_ids = {id(g) for g in globs_copy.values()}
+ for stack_element in _postproc:
+ if stack_element in glob_ids:
+ _postproc[stack_element].append((_setitems, (globs, globs_copy)))
+ break
+ else:
+ postproc_list.append((_setitems, (globs, globs_copy)))
+
+ closure = obj.__closure__
+ state_dict = {}
+ for fattrname in ('__doc__', '__kwdefaults__', '__annotations__'):
+ fattr = getattr(obj, fattrname, None)
+ if fattr is not None:
+ state_dict[fattrname] = fattr
+ if obj.__qualname__ != obj.__name__:
+ state_dict['__qualname__'] = obj.__qualname__
+ if '__name__' not in globs or obj.__module__ != globs['__name__']:
+ state_dict['__module__'] = obj.__module__
+
+ state = obj.__dict__
+ if type(state) is not dict:
+ state_dict['__dict__'] = state
+ state = None
+ if state_dict:
+ state = state, state_dict
+
+ _save_with_postproc(pickler, (_create_function, (
+ obj.__code__, globs, obj.__name__, obj.__defaults__,
+ closure
+ ), state), obj=obj, postproc_list=postproc_list)
+
+ # Lift closure cell update to earliest function (#458)
+ if _postproc:
+ topmost_postproc = next(iter(_postproc.values()), None)
+ if closure and topmost_postproc:
+ for cell in closure:
+ possible_postproc = (setattr, (cell, 'cell_contents', obj))
+ try:
+ topmost_postproc.remove(possible_postproc)
+ except ValueError:
+ continue
+
+ # Change the value of the cell
+ pickler.save_reduce(*possible_postproc)
+ # pop None created by calling preprocessing step off stack
+ pickler.write(POP)
+
+ logger.trace(pickler, "# F1")
+ else:
+ logger.trace(pickler, "F2: %s", obj)
+ name = getattr(obj, '__qualname__', getattr(obj, '__name__', None))
+ StockPickler.save_global(pickler, obj, name=name)
+ logger.trace(pickler, "# F2")
+ return
+
+if HAS_CTYPES and hasattr(ctypes, 'pythonapi'):
+ _PyCapsule_New = ctypes.pythonapi.PyCapsule_New
+ _PyCapsule_New.argtypes = (ctypes.c_void_p, ctypes.c_char_p, ctypes.c_void_p)
+ _PyCapsule_New.restype = ctypes.py_object
+ _PyCapsule_GetPointer = ctypes.pythonapi.PyCapsule_GetPointer
+ _PyCapsule_GetPointer.argtypes = (ctypes.py_object, ctypes.c_char_p)
+ _PyCapsule_GetPointer.restype = ctypes.c_void_p
+ _PyCapsule_GetDestructor = ctypes.pythonapi.PyCapsule_GetDestructor
+ _PyCapsule_GetDestructor.argtypes = (ctypes.py_object,)
+ _PyCapsule_GetDestructor.restype = ctypes.c_void_p
+ _PyCapsule_GetContext = ctypes.pythonapi.PyCapsule_GetContext
+ _PyCapsule_GetContext.argtypes = (ctypes.py_object,)
+ _PyCapsule_GetContext.restype = ctypes.c_void_p
+ _PyCapsule_GetName = ctypes.pythonapi.PyCapsule_GetName
+ _PyCapsule_GetName.argtypes = (ctypes.py_object,)
+ _PyCapsule_GetName.restype = ctypes.c_char_p
+ _PyCapsule_IsValid = ctypes.pythonapi.PyCapsule_IsValid
+ _PyCapsule_IsValid.argtypes = (ctypes.py_object, ctypes.c_char_p)
+ _PyCapsule_IsValid.restype = ctypes.c_bool
+ _PyCapsule_SetContext = ctypes.pythonapi.PyCapsule_SetContext
+ _PyCapsule_SetContext.argtypes = (ctypes.py_object, ctypes.c_void_p)
+ _PyCapsule_SetDestructor = ctypes.pythonapi.PyCapsule_SetDestructor
+ _PyCapsule_SetDestructor.argtypes = (ctypes.py_object, ctypes.c_void_p)
+ _PyCapsule_SetName = ctypes.pythonapi.PyCapsule_SetName
+ _PyCapsule_SetName.argtypes = (ctypes.py_object, ctypes.c_char_p)
+ _PyCapsule_SetPointer = ctypes.pythonapi.PyCapsule_SetPointer
+ _PyCapsule_SetPointer.argtypes = (ctypes.py_object, ctypes.c_void_p)
+ #from _socket import CAPI as _testcapsule
+ _testcapsule_name = b'dill._dill._testcapsule'
+ _testcapsule = _PyCapsule_New(
+ ctypes.cast(_PyCapsule_New, ctypes.c_void_p),
+ ctypes.c_char_p(_testcapsule_name),
+ None
+ )
+ PyCapsuleType = type(_testcapsule)
+ @register(PyCapsuleType)
+ def save_capsule(pickler, obj):
+ logger.trace(pickler, "Cap: %s", obj)
+ name = _PyCapsule_GetName(obj)
+ #warnings.warn('Pickling a PyCapsule (%s) does not pickle any C data structures and could cause segmentation faults or other memory errors when unpickling.' % (name,), PicklingWarning)
+ pointer = _PyCapsule_GetPointer(obj, name)
+ context = _PyCapsule_GetContext(obj)
+ destructor = _PyCapsule_GetDestructor(obj)
+ pickler.save_reduce(_create_capsule, (pointer, name, context, destructor), obj=obj)
+ logger.trace(pickler, "# Cap")
+ _incedental_reverse_typemap['PyCapsuleType'] = PyCapsuleType
+ _reverse_typemap['PyCapsuleType'] = PyCapsuleType
+ _incedental_types.add(PyCapsuleType)
+else:
+ _testcapsule = None
+
+@register(ContextType)
+def save_context(pickler, obj):
+ logger.trace(pickler, "Cx: %s", obj)
+ pickler.save_reduce(ContextType, tuple(obj.items()), obj=obj)
+ logger.trace(pickler, "# Cx")
+
+
+#############################
+# A quick fix for issue #500
+# This should be removed when a better solution is found.
+
+if hasattr(dataclasses, "_HAS_DEFAULT_FACTORY_CLASS"):
+ @register(dataclasses._HAS_DEFAULT_FACTORY_CLASS)
+ def save_dataclasses_HAS_DEFAULT_FACTORY_CLASS(pickler, obj):
+ logger.trace(pickler, "DcHDF: %s", obj)
+ pickler.write(GLOBAL + b"dataclasses\n_HAS_DEFAULT_FACTORY\n")
+ logger.trace(pickler, "# DcHDF")
+
+if hasattr(dataclasses, "MISSING"):
+ @register(type(dataclasses.MISSING))
+ def save_dataclasses_MISSING_TYPE(pickler, obj):
+ logger.trace(pickler, "DcM: %s", obj)
+ pickler.write(GLOBAL + b"dataclasses\nMISSING\n")
+ logger.trace(pickler, "# DcM")
+
+if hasattr(dataclasses, "KW_ONLY"):
+ @register(type(dataclasses.KW_ONLY))
+ def save_dataclasses_KW_ONLY_TYPE(pickler, obj):
+ logger.trace(pickler, "DcKWO: %s", obj)
+ pickler.write(GLOBAL + b"dataclasses\nKW_ONLY\n")
+ logger.trace(pickler, "# DcKWO")
+
+if hasattr(dataclasses, "_FIELD_BASE"):
+ @register(dataclasses._FIELD_BASE)
+ def save_dataclasses_FIELD_BASE(pickler, obj):
+ logger.trace(pickler, "DcFB: %s", obj)
+ pickler.write(GLOBAL + b"dataclasses\n" + obj.name.encode() + b"\n")
+ logger.trace(pickler, "# DcFB")
+
+#############################
+
+# quick sanity checking
+def pickles(obj,exact=False,safe=False,**kwds):
+ """
+ Quick check if object pickles with dill.
+
+ If *exact=True* then an equality test is done to check if the reconstructed
+ object matches the original object.
+
+ If *safe=True* then any exception will raised in copy signal that the
+ object is not picklable, otherwise only pickling errors will be trapped.
+
+ Additional keyword arguments are as :func:`dumps` and :func:`loads`.
+ """
+ if safe: exceptions = (Exception,) # RuntimeError, ValueError
+ else:
+ exceptions = (TypeError, AssertionError, NotImplementedError, PicklingError, UnpicklingError)
+ try:
+ pik = copy(obj, **kwds)
+ #FIXME: should check types match first, then check content if "exact"
+ try:
+ #FIXME: should be "(pik == obj).all()" for numpy comparison, though that'll fail if shapes differ
+ result = bool(pik.all() == obj.all())
+ except (AttributeError, TypeError):
+ warnings.filterwarnings('ignore') #FIXME: be specific
+ result = pik == obj
+ if warnings.filters: del warnings.filters[0]
+ if hasattr(result, 'toarray'): # for unusual types like sparse matrix
+ result = result.toarray().all()
+ if result: return True
+ if not exact:
+ result = type(pik) == type(obj)
+ if result: return result
+ # class instances might have been dumped with byref=False
+ return repr(type(pik)) == repr(type(obj)) #XXX: InstanceType?
+ return False
+ except exceptions:
+ return False
+
+def check(obj, *args, **kwds):
+ """
+ Check pickling of an object across another process.
+
+ *python* is the path to the python interpreter (defaults to sys.executable)
+
+ Set *verbose=True* to print the unpickled object in the other process.
+
+ Additional keyword arguments are as :func:`dumps` and :func:`loads`.
+ """
+ # == undocumented ==
+ # python -- the string path or executable name of the selected python
+ # verbose -- if True, be verbose about printing warning messages
+ # all other args and kwds are passed to dill.dumps #FIXME: ignore on load
+ verbose = kwds.pop('verbose', False)
+ python = kwds.pop('python', None)
+ if python is None:
+ import sys
+ python = sys.executable
+ # type check
+ isinstance(python, str)
+ import subprocess
+ fail = True
+ try:
+ _obj = dumps(obj, *args, **kwds)
+ fail = False
+ finally:
+ if fail and verbose:
+ print("DUMP FAILED")
+ #FIXME: fails if python interpreter path contains spaces
+ # Use the following instead (which also processes the 'ignore' keyword):
+ # ignore = kwds.pop('ignore', None)
+ # unpickle = "dill.loads(%s, ignore=%s)"%(repr(_obj), repr(ignore))
+ # cmd = [python, "-c", "import dill; print(%s)"%unpickle]
+ # msg = "SUCCESS" if not subprocess.call(cmd) else "LOAD FAILED"
+ msg = "%s -c import dill; print(dill.loads(%s))" % (python, repr(_obj))
+ msg = "SUCCESS" if not subprocess.call(msg.split(None,2)) else "LOAD FAILED"
+ if verbose:
+ print(msg)
+ return
+
+# use to protect against missing attributes
+def is_dill(pickler, child=None):
+ "check the dill-ness of your pickler"
+ if child is False or not hasattr(pickler.__class__, 'mro'):
+ return 'dill' in pickler.__module__
+ return Pickler in pickler.__class__.mro()
+
+def _extend():
+ """extend pickle with all of dill's registered types"""
+ # need to have pickle not choke on _main_module? use is_dill(pickler)
+ for t,func in Pickler.dispatch.items():
+ try:
+ StockPickler.dispatch[t] = func
+ except Exception: #TypeError, PicklingError, UnpicklingError
+ logger.trace(pickler, "skip: %s", t)
+ return
+
+del diff, _use_diff, use_diff
+
+# EOF
diff --git a/venv/Lib/site-packages/dill/_objects.py b/venv/Lib/site-packages/dill/_objects.py
new file mode 100644
index 0000000000000000000000000000000000000000..573fae85bcb1d043e5f07bd57d4d3ae240206424
--- /dev/null
+++ b/venv/Lib/site-packages/dill/_objects.py
@@ -0,0 +1,545 @@
+#!/usr/bin/env python
+#
+# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
+# Copyright (c) 2008-2016 California Institute of Technology.
+# Copyright (c) 2016-2026 The Uncertainty Quantification Foundation.
+# License: 3-clause BSD. The full license text is available at:
+# - https://github.com/uqfoundation/dill/blob/master/LICENSE
+"""
+all Python Standard Library objects (currently: CH 1-15 @ 2.7)
+and some other common objects (i.e. numpy.ndarray)
+"""
+
+__all__ = ['registered','failures','succeeds']
+
+# helper imports
+import warnings; warnings.filterwarnings("ignore", category=DeprecationWarning)
+import sys
+import queue as Queue
+#import dbm as anydbm #XXX: delete foo
+from io import BytesIO as StringIO
+import re
+import array
+import collections
+import codecs
+import struct
+import dataclasses
+import datetime
+import calendar
+import weakref
+import pprint
+import decimal
+import numbers
+import functools
+import itertools
+import operator
+import tempfile
+import shelve
+import zlib
+import gzip
+import zipfile
+import tarfile
+import csv
+import hashlib
+import hmac
+import os
+import logging
+import logging.handlers
+import optparse
+#import __hello__
+import threading
+import socket
+import contextlib
+import contextvars
+try:
+ import bz2
+ import sqlite3
+ import dbm.ndbm as dbm
+ HAS_ALL = True
+except ImportError: # Ubuntu
+ HAS_ALL = False
+try:
+ #import curses
+ #from curses import textpad, panel
+ HAS_CURSES = True
+except ImportError: # Windows
+ HAS_CURSES = False
+try:
+ import ctypes
+ HAS_CTYPES = True
+ # if using `pypy`, pythonapi is not found
+ IS_PYPY = not hasattr(ctypes, 'pythonapi')
+except ImportError: # MacPorts
+ HAS_CTYPES = False
+ IS_PYPY = False
+
+IS_PYODIDE = sys.platform == 'emscripten'
+
+# helper objects
+class _class:
+ def _method(self):
+ pass
+# @classmethod
+# def _clsmethod(cls): #XXX: test me
+# pass
+# @staticmethod
+# def _static(self): #XXX: test me
+# pass
+class _class2:
+ def __call__(self):
+ pass
+_instance2 = _class2()
+class _newclass(object):
+ def _method(self):
+ pass
+# @classmethod
+# def _clsmethod(cls): #XXX: test me
+# pass
+# @staticmethod
+# def _static(self): #XXX: test me
+# pass
+class _newclass2(object):
+ __slots__ = ['descriptor']
+def _function(x): yield x
+def _function2():
+ try: raise
+ except Exception:
+ from sys import exc_info
+ e, er, tb = exc_info()
+ return er, tb
+if HAS_CTYPES:
+ class _Struct(ctypes.Structure):
+ pass
+ _Struct._fields_ = [("_field", ctypes.c_int),("next", ctypes.POINTER(_Struct))]
+_filedescrip, _tempfile = tempfile.mkstemp('r') # deleted in cleanup
+if sys.hexversion < 0x30d00a1:
+ _tmpf = tempfile.TemporaryFile('w') # emits OSError 9 in python 3.13
+else:
+ _tmpf = tempfile.NamedTemporaryFile('w').file # for > python 3.9
+
+# objects used by dill for type declaration
+registered = d = {}
+# objects dill fails to pickle
+failures = x = {}
+# all other type objects
+succeeds = a = {}
+
+# types module (part of CH 8)
+a['BooleanType'] = bool(1)
+a['BuiltinFunctionType'] = len
+a['BuiltinMethodType'] = a['BuiltinFunctionType']
+a['BytesType'] = _bytes = codecs.latin_1_encode('\x00')[0] # bytes(1)
+a['ClassType'] = _class
+a['ComplexType'] = complex(1)
+a['DictType'] = _dict = {}
+a['DictionaryType'] = a['DictType']
+a['FloatType'] = float(1)
+a['FunctionType'] = _function
+a['InstanceType'] = _instance = _class()
+a['IntType'] = _int = int(1)
+a['ListType'] = _list = []
+a['NoneType'] = None
+a['ObjectType'] = object()
+a['StringType'] = _str = str(1)
+a['TupleType'] = _tuple = ()
+a['TypeType'] = type
+a['LongType'] = _int
+a['UnicodeType'] = _str
+# built-in constants (CH 4)
+a['CopyrightType'] = copyright
+# built-in types (CH 5)
+a['ClassObjectType'] = _newclass #
+a['ClassInstanceType'] = _newclass() #
+a['SetType'] = _set = set()
+a['FrozenSetType'] = frozenset()
+# built-in exceptions (CH 6)
+a['ExceptionType'] = _exception = _function2()[0]
+# string services (CH 7)
+a['SREPatternType'] = _srepattern = re.compile('')
+# data types (CH 8)
+a['ArrayType'] = array.array("f")
+a['DequeType'] = collections.deque([0])
+a['DefaultDictType'] = collections.defaultdict(_function, _dict)
+a['TZInfoType'] = datetime.tzinfo()
+a['DateTimeType'] = datetime.datetime.today()
+a['CalendarType'] = calendar.Calendar()
+# numeric and mathematical types (CH 9)
+a['DecimalType'] = decimal.Decimal(1)
+# data compression and archiving (CH 12)
+a['TarInfoType'] = tarfile.TarInfo()
+# generic operating system services (CH 15)
+a['LoggerType'] = _logger = logging.getLogger()
+a['FormatterType'] = logging.Formatter() # pickle ok
+a['FilterType'] = logging.Filter() # pickle ok
+a['LogRecordType'] = logging.makeLogRecord(_dict) # pickle ok
+a['OptionParserType'] = _oparser = optparse.OptionParser() # pickle ok
+a['OptionGroupType'] = optparse.OptionGroup(_oparser,"foo") # pickle ok
+a['OptionType'] = optparse.Option('--foo') # pickle ok
+if HAS_CTYPES:
+ z = x if (IS_PYPY and sys.hexversion < 0x30b0df0) else a
+ z['CCharType'] = _cchar = ctypes.c_char()
+ z['CWCharType'] = ctypes.c_wchar() # fail == 2.6
+ z['CByteType'] = ctypes.c_byte()
+ z['CUByteType'] = ctypes.c_ubyte()
+ z['CShortType'] = ctypes.c_short()
+ z['CUShortType'] = ctypes.c_ushort()
+ z['CIntType'] = ctypes.c_int()
+ z['CUIntType'] = ctypes.c_uint()
+ z['CLongType'] = ctypes.c_long()
+ z['CULongType'] = ctypes.c_ulong()
+ z['CLongLongType'] = ctypes.c_longlong()
+ z['CULongLongType'] = ctypes.c_ulonglong()
+ z['CFloatType'] = ctypes.c_float()
+ z['CDoubleType'] = ctypes.c_double()
+ z['CSizeTType'] = ctypes.c_size_t()
+ del z
+ a['CLibraryLoaderType'] = ctypes.cdll
+ a['StructureType'] = _Struct
+ # if not IS_PYPY:
+ # a['BigEndianStructureType'] = ctypes.BigEndianStructure()
+#NOTE: also LittleEndianStructureType and UnionType... abstract classes
+#NOTE: remember for ctypesobj.contents creates a new python object
+#NOTE: ctypes.c_int._objects is memberdescriptor for object's __dict__
+#NOTE: base class of all ctypes data types is non-public _CData
+
+import fractions
+import io
+from io import StringIO as TextIO
+# built-in functions (CH 2)
+a['ByteArrayType'] = bytearray([1])
+# numeric and mathematical types (CH 9)
+a['FractionType'] = fractions.Fraction()
+a['NumberType'] = numbers.Number()
+# generic operating system services (CH 15)
+a['IOBaseType'] = io.IOBase()
+a['RawIOBaseType'] = io.RawIOBase()
+a['TextIOBaseType'] = io.TextIOBase()
+a['BufferedIOBaseType'] = io.BufferedIOBase()
+a['UnicodeIOType'] = TextIO() # the new StringIO
+a['LoggerAdapterType'] = logging.LoggerAdapter(_logger,_dict) # pickle ok
+if HAS_CTYPES:
+ z = x if (IS_PYPY and sys.hexversion < 0x30b0df0) else a
+ z['CBoolType'] = ctypes.c_bool(1)
+ z['CLongDoubleType'] = ctypes.c_longdouble()
+ del z
+import argparse
+# data types (CH 8)
+a['OrderedDictType'] = collections.OrderedDict(_dict)
+a['CounterType'] = collections.Counter(_dict)
+if HAS_CTYPES:
+ z = x if (IS_PYPY and sys.hexversion < 0x30b0df0) else a
+ z['CSSizeTType'] = ctypes.c_ssize_t()
+ del z
+# generic operating system services (CH 15)
+a['NullHandlerType'] = logging.NullHandler() # pickle ok # new 2.7
+a['ArgParseFileType'] = argparse.FileType() # pickle ok
+
+# -- pickle fails on all below here -----------------------------------------
+# types module (part of CH 8)
+a['CodeType'] = compile('','','exec')
+a['DictProxyType'] = type.__dict__
+a['DictProxyType2'] = _newclass.__dict__
+a['EllipsisType'] = Ellipsis
+a['ClosedFileType'] = open(os.devnull, 'wb', buffering=0).close()
+a['GetSetDescriptorType'] = array.array.typecode
+a['LambdaType'] = _lambda = lambda x: lambda y: x #XXX: works when not imported!
+a['MemberDescriptorType'] = _newclass2.descriptor
+if not IS_PYPY:
+ a['MemberDescriptorType2'] = datetime.timedelta.days
+a['MethodType'] = _method = _class()._method #XXX: works when not imported!
+a['ModuleType'] = datetime
+a['NotImplementedType'] = NotImplemented
+a['SliceType'] = slice(1)
+a['UnboundMethodType'] = _class._method #XXX: works when not imported!
+from dill._dill import get_file_type as openfile
+d['TextWrapperType'] = openfile('r', buffering=-1) # same as mode='w','w+','r+'
+if not IS_PYODIDE:
+ d['BufferedRandomType'] = openfile('r+b', buffering=-1) # same as mode='w+b'
+d['BufferedReaderType'] = openfile('rb', buffering=-1) # (default: buffering=-1)
+d['BufferedWriterType'] = openfile('wb', buffering=-1)
+try: # oddities: deprecated
+ from _pyio import open as _open
+ d['PyTextWrapperType'] = openfile('r', buffering=-1, open=_open)
+ if not IS_PYODIDE:
+ d['PyBufferedRandomType'] = openfile('r+b', buffering=-1, open=_open)
+ d['PyBufferedReaderType'] = openfile('rb', buffering=-1, open=_open)
+ d['PyBufferedWriterType'] = openfile('wb', buffering=-1, open=_open)
+except ImportError:
+ pass
+del openfile
+# other (concrete) object types
+z = d if sys.hexversion < 0x30800a2 else a
+z['CellType'] = (_lambda)(0).__closure__[0]
+del z
+a['XRangeType'] = _xrange = range(1)
+a['MethodDescriptorType'] = type.__dict__['mro']
+a['WrapperDescriptorType'] = type.__repr__
+#a['WrapperDescriptorType2'] = type.__dict__['__module__']#XXX: GetSetDescriptor
+a['ClassMethodDescriptorType'] = type.__dict__['__prepare__']
+# built-in functions (CH 2)
+_methodwrap = (1).__lt__
+a['MethodWrapperType'] = _methodwrap
+a['StaticMethodType'] = staticmethod(_method)
+a['ClassMethodType'] = classmethod(_method)
+a['PropertyType'] = property()
+d['SuperType'] = super(Exception, _exception)
+# string services (CH 7)
+_in = _bytes
+a['InputType'] = _cstrI = StringIO(_in)
+a['OutputType'] = _cstrO = StringIO()
+# data types (CH 8)
+a['WeakKeyDictionaryType'] = weakref.WeakKeyDictionary()
+a['WeakValueDictionaryType'] = weakref.WeakValueDictionary()
+a['ReferenceType'] = weakref.ref(_instance)
+a['DeadReferenceType'] = weakref.ref(_class())
+a['ProxyType'] = weakref.proxy(_instance)
+a['DeadProxyType'] = weakref.proxy(_class())
+a['CallableProxyType'] = weakref.proxy(_instance2)
+a['DeadCallableProxyType'] = weakref.proxy(_class2())
+a['QueueType'] = Queue.Queue()
+# numeric and mathematical types (CH 9)
+d['PartialType'] = functools.partial(int,base=2)
+a['IzipType'] = zip('0','1')
+d['ItemGetterType'] = operator.itemgetter(0)
+d['AttrGetterType'] = operator.attrgetter('__repr__')
+# file and directory access (CH 10)
+_fileW = _cstrO
+# data persistence (CH 11)
+if HAS_ALL:
+ x['ConnectionType'] = _conn = sqlite3.connect(':memory:')
+ x['CursorType'] = _conn.cursor()
+a['ShelveType'] = shelve.Shelf({})
+# data compression and archiving (CH 12)
+if HAS_ALL:
+ x['BZ2FileType'] = bz2.BZ2File(os.devnull)
+ x['BZ2CompressorType'] = bz2.BZ2Compressor()
+ x['BZ2DecompressorType'] = bz2.BZ2Decompressor()
+#x['ZipFileType'] = _zip = zipfile.ZipFile(os.devnull,'w')
+#_zip.write(_tempfile,'x') [causes annoying warning/error printed on import]
+#a['ZipInfoType'] = _zip.getinfo('x')
+a['TarFileType'] = tarfile.open(fileobj=_fileW,mode='w')
+# file formats (CH 13)
+x['DialectType'] = csv.get_dialect('excel')
+if sys.hexversion < 0x30d00a1:
+ import xdrlib
+ a['PackerType'] = xdrlib.Packer()
+# optional operating system services (CH 16)
+a['LockType'] = threading.Lock()
+a['RLockType'] = threading.RLock()
+# generic operating system services (CH 15) # also closed/open and r/w/etc...
+a['NamedLoggerType'] = _logger = logging.getLogger(__name__)
+#a['FrozenModuleType'] = __hello__ #FIXME: prints "Hello world..."
+# interprocess communication (CH 17)
+x['SocketType'] = _socket = socket.socket()
+x['SocketPairType'] = socket.socketpair()[0]
+# python runtime services (CH 27)
+a['GeneratorContextManagerType'] = contextlib.contextmanager(max)([1])
+#a['ContextType'] = contextvars.Context() #XXX: ContextVar
+
+try: # ipython
+ __IPYTHON__ is True # is ipython
+except NameError:
+ # built-in constants (CH 4)
+ a['QuitterType'] = quit
+ d['ExitType'] = a['QuitterType']
+try: # numpy #FIXME: slow... 0.05 to 0.1 sec to import numpy
+ from numpy import ufunc as _numpy_ufunc
+ from numpy import array as _numpy_array
+ from numpy import int32 as _numpy_int32
+ a['NumpyUfuncType'] = _numpy_ufunc
+ a['NumpyArrayType'] = _numpy_array
+ a['NumpyInt32Type'] = _numpy_int32
+except ImportError:
+ pass
+# generic operating system services (CH 15)
+a['FileHandlerType'] = logging.FileHandler(os.devnull)
+a['RotatingFileHandlerType'] = logging.handlers.RotatingFileHandler(os.devnull)
+a['SocketHandlerType'] = logging.handlers.SocketHandler('localhost',514)
+a['MemoryHandlerType'] = logging.handlers.MemoryHandler(1)
+# data types (CH 8)
+a['WeakSetType'] = weakref.WeakSet() # 2.7
+# generic operating system services (CH 15) [errors when dill is imported]
+#a['ArgumentParserType'] = _parser = argparse.ArgumentParser('PROG')
+#a['NamespaceType'] = _parser.parse_args() # pickle ok
+#a['SubParsersActionType'] = _parser.add_subparsers()
+#a['MutuallyExclusiveGroupType'] = _parser.add_mutually_exclusive_group()
+#a['ArgumentGroupType'] = _parser.add_argument_group()
+
+# -- dill fails in some versions below here ---------------------------------
+# types module (part of CH 8)
+d['FileType'] = open(os.devnull, 'rb', buffering=0) # same 'wb','wb+','rb+'
+# built-in functions (CH 2)
+# Iterators:
+a['ListIteratorType'] = iter(_list) # empty vs non-empty
+a['SetIteratorType'] = iter(_set) #XXX: empty vs non-empty #FIXME: list_iterator
+a['TupleIteratorType']= iter(_tuple) # empty vs non-empty
+a['XRangeIteratorType'] = iter(_xrange) # empty vs non-empty
+a["BytesIteratorType"] = iter(b'')
+a["BytearrayIteratorType"] = iter(bytearray(b''))
+z = x if IS_PYPY else a
+z["CallableIteratorType"] = iter(iter, None)
+del z
+x["MemoryIteratorType"] = iter(memoryview(b''))
+a["ListReverseiteratorType"] = reversed([])
+X = a['OrderedDictType']
+d["OdictKeysType"] = X.keys()
+d["OdictValuesType"] = X.values()
+d["OdictItemsType"] = X.items()
+a["OdictIteratorType"] = iter(X.keys()) #FIXME: list_iterator
+del X
+#FIXME: list_iterator
+a['DictionaryItemIteratorType'] = iter(type.__dict__.items())
+a['DictionaryKeyIteratorType'] = iter(type.__dict__.keys())
+a['DictionaryValueIteratorType'] = iter(type.__dict__.values())
+if sys.hexversion >= 0x30800a0:
+ a["DictReversekeyiteratorType"] = reversed({}.keys())
+ a["DictReversevalueiteratorType"] = reversed({}.values())
+ a["DictReverseitemiteratorType"] = reversed({}.items())
+
+try:
+ import symtable
+ #FIXME: fails to pickle
+ x["SymtableEntryType"] = symtable.symtable("", "string", "exec")._table
+except ImportError:
+ pass
+
+if sys.hexversion >= 0x30a00a0 and not IS_PYPY:
+ x['LineIteratorType'] = compile('3', '', 'eval').co_lines()
+
+if sys.hexversion >= 0x30b00b0 and not IS_PYPY:
+ from types import GenericAlias
+ d["GenericAliasIteratorType"] = iter(GenericAlias(list, (int,)))
+ x['PositionsIteratorType'] = compile('3', '', 'eval').co_positions()
+
+# data types (CH 8)
+a['PrettyPrinterType'] = pprint.PrettyPrinter()
+# file and directory access (CH 10)
+a['TemporaryFileType'] = _tmpf
+# data compression and archiving (CH 12)
+x['GzipFileType'] = gzip.GzipFile(fileobj=_fileW)
+# generic operating system services (CH 15)
+a['StreamHandlerType'] = logging.StreamHandler()
+# numeric and mathematical types (CH 9)
+z = a if sys.hexversion < 0x30e00a1 else x
+z['CountType'] = itertools.count(0) #FIXME: __reduce__ removed in 3.14.0a1
+z['ChainType'] = itertools.chain('0','1')
+z['ProductType'] = itertools.product('0','1')
+z['CycleType'] = itertools.cycle('0')
+z['PermutationsType'] = itertools.permutations('0')
+z['CombinationsType'] = itertools.combinations('0',1)
+z['RepeatType'] = itertools.repeat(0)
+z['CompressType'] = itertools.compress('0',[1])
+del z
+#XXX: ...and etc
+
+# -- dill fails on all below here -------------------------------------------
+# types module (part of CH 8)
+x['GeneratorType'] = _generator = _function(1) #XXX: priority
+x['FrameType'] = _generator.gi_frame #XXX: inspect.currentframe()
+x['TracebackType'] = _function2()[1] #(see: inspect.getouterframes,getframeinfo)
+# other (concrete) object types
+# (also: Capsule / CObject ?)
+# built-in functions (CH 2)
+# built-in types (CH 5)
+# string services (CH 7)
+x['StructType'] = struct.Struct('c')
+x['CallableIteratorType'] = _srepattern.finditer('')
+x['SREMatchType'] = _srepattern.match('')
+x['SREScannerType'] = _srepattern.scanner('')
+x['StreamReader'] = codecs.StreamReader(_cstrI) #XXX: ... and etc
+# python object persistence (CH 11)
+# x['DbShelveType'] = shelve.open('foo','n')#,protocol=2) #XXX: delete foo
+if HAS_ALL:
+ z = a if IS_PYPY else x
+ z['DbmType'] = dbm.open(_tempfile,'n')
+ del z
+# x['DbCursorType'] = _dbcursor = anydbm.open('foo','n') #XXX: delete foo
+# x['DbType'] = _dbcursor.db
+# data compression and archiving (CH 12)
+x['ZlibCompressType'] = zlib.compressobj()
+x['ZlibDecompressType'] = zlib.decompressobj()
+# file formats (CH 13)
+x['CSVReaderType'] = csv.reader(_cstrI)
+x['CSVWriterType'] = csv.writer(_cstrO)
+x['CSVDictReaderType'] = csv.DictReader(_cstrI)
+x['CSVDictWriterType'] = csv.DictWriter(_cstrO,{})
+# cryptographic services (CH 14)
+x['HashType'] = hashlib.md5()
+if (sys.hexversion < 0x30800a1):
+ x['HMACType'] = hmac.new(_in)
+else:
+ x['HMACType'] = hmac.new(_in, digestmod='md5')
+# generic operating system services (CH 15)
+if HAS_CURSES: pass
+ #x['CursesWindowType'] = _curwin = curses.initscr() #FIXME: messes up tty
+ #x['CursesTextPadType'] = textpad.Textbox(_curwin)
+ #x['CursesPanelType'] = panel.new_panel(_curwin)
+if HAS_CTYPES:
+ x['CCharPType'] = ctypes.c_char_p()
+ x['CWCharPType'] = ctypes.c_wchar_p()
+ x['CVoidPType'] = ctypes.c_void_p()
+ if sys.platform[:3] == 'win':
+ x['CDLLType'] = _cdll = ctypes.cdll.msvcrt
+ else:
+ x['CDLLType'] = _cdll = ctypes.CDLL(None)
+ if not IS_PYPY:
+ x['PyDLLType'] = _pydll = ctypes.pythonapi
+ x['FuncPtrType'] = _cdll._FuncPtr()
+ x['CCharArrayType'] = ctypes.create_string_buffer(1)
+ x['CWCharArrayType'] = ctypes.create_unicode_buffer(1)
+ x['CParamType'] = ctypes.byref(_cchar)
+ x['LPCCharType'] = ctypes.pointer(_cchar)
+ x['LPCCharObjType'] = _lpchar = ctypes.POINTER(ctypes.c_char)
+ x['NullPtrType'] = _lpchar()
+ x['NullPyObjectType'] = ctypes.py_object()
+ x['PyObjectType'] = ctypes.py_object(lambda :None)
+ z = a if IS_PYPY else x
+ z['FieldType'] = _field = _Struct._field
+ z['CFUNCTYPEType'] = _cfunc = ctypes.CFUNCTYPE(ctypes.c_char)
+ if sys.hexversion < 0x30c00b3:
+ x['CFunctionType'] = _cfunc(str)
+ del z
+# numeric and mathematical types (CH 9)
+a['MethodCallerType'] = operator.methodcaller('mro') # 2.6
+# built-in types (CH 5)
+x['MemoryType'] = memoryview(_in) # 2.7
+x['MemoryType2'] = memoryview(bytearray(_in)) # 2.7
+d['DictItemsType'] = _dict.items() # 2.7
+d['DictKeysType'] = _dict.keys() # 2.7
+d['DictValuesType'] = _dict.values() # 2.7
+# generic operating system services (CH 15)
+a['RawTextHelpFormatterType'] = argparse.RawTextHelpFormatter('PROG')
+a['RawDescriptionHelpFormatterType'] = argparse.RawDescriptionHelpFormatter('PROG')
+a['ArgDefaultsHelpFormatterType'] = argparse.ArgumentDefaultsHelpFormatter('PROG')
+z = a if IS_PYPY else x
+z['CmpKeyType'] = _cmpkey = functools.cmp_to_key(_methodwrap) # 2.7, >=3.2
+z['CmpKeyObjType'] = _cmpkey('0') #2.7, >=3.2
+del z
+# oddities: removed, etc
+x['BufferType'] = x['MemoryType']
+
+from dill._dill import _testcapsule
+if _testcapsule is not None:
+ d['PyCapsuleType'] = _testcapsule
+del _testcapsule
+
+if hasattr(dataclasses, '_HAS_DEFAULT_FACTORY'):
+ a['DataclassesHasDefaultFactoryType'] = dataclasses._HAS_DEFAULT_FACTORY
+
+if hasattr(dataclasses, 'MISSING'):
+ a['DataclassesMissingType'] = dataclasses.MISSING
+
+if hasattr(dataclasses, 'KW_ONLY'):
+ a['DataclassesKWOnlyType'] = dataclasses.KW_ONLY
+
+if hasattr(dataclasses, '_FIELD_BASE'):
+ a['DataclassesFieldBaseType'] = dataclasses._FIELD
+
+# -- cleanup ----------------------------------------------------------------
+a.update(d) # registered also succeed
+if sys.platform[:3] == 'win':
+ os.close(_filedescrip) # required on win32
+os.remove(_tempfile)
+
+
+# EOF
diff --git a/venv/Lib/site-packages/dill/_shims.py b/venv/Lib/site-packages/dill/_shims.py
new file mode 100644
index 0000000000000000000000000000000000000000..a7a5bdf2f7c9dd6d05882337b7ad462a079551fc
--- /dev/null
+++ b/venv/Lib/site-packages/dill/_shims.py
@@ -0,0 +1,193 @@
+#!/usr/bin/env python
+#
+# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
+# Author: Anirudh Vegesana (avegesan@cs.stanford.edu)
+# Copyright (c) 2021-2026 The Uncertainty Quantification Foundation.
+# License: 3-clause BSD. The full license text is available at:
+# - https://github.com/uqfoundation/dill/blob/master/LICENSE
+"""
+Provides shims for compatibility between versions of dill and Python.
+
+Compatibility shims should be provided in this file. Here are two simple example
+use cases.
+
+Deprecation of constructor function:
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Assume that we were transitioning _import_module in _dill.py to
+the builtin function importlib.import_module when present.
+
+@move_to(_dill)
+def _import_module(import_name):
+ ... # code already in _dill.py
+
+_import_module = Getattr(importlib, 'import_module', Getattr(_dill, '_import_module', None))
+
+The code will attempt to find import_module in the importlib module. If not
+present, it will use the _import_module function in _dill.
+
+Emulate new Python behavior in older Python versions:
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+CellType.cell_contents behaves differently in Python 3.6 and 3.7. It is
+read-only in Python 3.6 and writable and deletable in 3.7.
+
+if _dill.OLD37 and _dill.HAS_CTYPES and ...:
+ @move_to(_dill)
+ def _setattr(object, name, value):
+ if type(object) is _dill.CellType and name == 'cell_contents':
+ _PyCell_Set.argtypes = (ctypes.py_object, ctypes.py_object)
+ _PyCell_Set(object, value)
+ else:
+ setattr(object, name, value)
+... # more cases below
+
+_setattr = Getattr(_dill, '_setattr', setattr)
+
+_dill._setattr will be used when present to emulate Python 3.7 functionality in
+older versions of Python while defaulting to the standard setattr in 3.7+.
+
+See this PR for the discussion that lead to this system:
+https://github.com/uqfoundation/dill/pull/443
+"""
+
+import inspect
+import sys
+
+_dill = sys.modules['dill._dill']
+
+
+class Reduce(object):
+ """
+ Reduce objects are wrappers used for compatibility enforcement during
+ unpickle-time. They should only be used in calls to pickler.save and
+ other Reduce objects. They are only evaluated within unpickler.load.
+
+ Pickling a Reduce object makes the two implementations equivalent:
+
+ pickler.save(Reduce(*reduction))
+
+ pickler.save_reduce(*reduction, obj=reduction)
+ """
+ __slots__ = ['reduction']
+ def __new__(cls, *reduction, **kwargs):
+ """
+ Args:
+ *reduction: a tuple that matches the format given here:
+ https://docs.python.org/3/library/pickle.html#object.__reduce__
+ is_callable: a bool to indicate that the object created by
+ unpickling `reduction` is callable. If true, the current Reduce
+ is allowed to be used as the function in further save_reduce calls
+ or Reduce objects.
+ """
+ is_callable = kwargs.get('is_callable', False) # Pleases Py2. Can be removed later
+ if is_callable:
+ self = object.__new__(_CallableReduce)
+ else:
+ self = object.__new__(Reduce)
+ self.reduction = reduction
+ return self
+ def __repr__(self):
+ return 'Reduce%s' % (self.reduction,)
+ def __copy__(self):
+ return self # pragma: no cover
+ def __deepcopy__(self, memo):
+ return self # pragma: no cover
+ def __reduce__(self):
+ return self.reduction
+ def __reduce_ex__(self, protocol):
+ return self.__reduce__()
+
+class _CallableReduce(Reduce):
+ # A version of Reduce for functions. Used to trick pickler.save_reduce into
+ # thinking that Reduce objects of functions are themselves meaningful functions.
+ def __call__(self, *args, **kwargs):
+ reduction = self.__reduce__()
+ func = reduction[0]
+ f_args = reduction[1]
+ obj = func(*f_args)
+ return obj(*args, **kwargs)
+
+__NO_DEFAULT = _dill.Sentinel('Getattr.NO_DEFAULT')
+
+def Getattr(object, name, default=__NO_DEFAULT):
+ """
+ A Reduce object that represents the getattr operation. When unpickled, the
+ Getattr will access an attribute 'name' of 'object' and return the value
+ stored there. If the attribute doesn't exist, the default value will be
+ returned if present.
+
+ The following statements are equivalent:
+
+ Getattr(collections, 'OrderedDict')
+ Getattr(collections, 'spam', None)
+ Getattr(*args)
+
+ Reduce(getattr, (collections, 'OrderedDict'))
+ Reduce(getattr, (collections, 'spam', None))
+ Reduce(getattr, args)
+
+ During unpickling, the first two will result in collections.OrderedDict and
+ None respectively because the first attribute exists and the second one does
+ not, forcing it to use the default value given in the third argument.
+ """
+
+ if default is Getattr.NO_DEFAULT:
+ reduction = (getattr, (object, name))
+ else:
+ reduction = (getattr, (object, name, default))
+
+ return Reduce(*reduction, is_callable=callable(default))
+
+Getattr.NO_DEFAULT = __NO_DEFAULT
+del __NO_DEFAULT
+
+def move_to(module, name=None):
+ def decorator(func):
+ if name is None:
+ fname = func.__name__
+ else:
+ fname = name
+ module.__dict__[fname] = func
+ func.__module__ = module.__name__
+ return func
+ return decorator
+
+def register_shim(name, default):
+ """
+ A easier to understand and more compact way of "softly" defining a function.
+ These two pieces of code are equivalent:
+
+ if _dill.OLD3X:
+ def _create_class():
+ ...
+ _create_class = register_shim('_create_class', types.new_class)
+
+ if _dill.OLD3X:
+ @move_to(_dill)
+ def _create_class():
+ ...
+ _create_class = Getattr(_dill, '_create_class', types.new_class)
+
+ Intuitively, it creates a function or object in the versions of dill/python
+ that require special reimplementations, and use a core library or default
+ implementation if that function or object does not exist.
+ """
+ func = globals().get(name)
+ if func is not None:
+ _dill.__dict__[name] = func
+ func.__module__ = _dill.__name__
+
+ if default is Getattr.NO_DEFAULT:
+ reduction = (getattr, (_dill, name))
+ else:
+ reduction = (getattr, (_dill, name, default))
+
+ return Reduce(*reduction, is_callable=callable(default))
+
+######################
+## Compatibility Shims are defined below
+######################
+
+_CELL_EMPTY = register_shim('_CELL_EMPTY', None)
+
+_setattr = register_shim('_setattr', setattr)
+_delattr = register_shim('_delattr', delattr)
diff --git a/venv/Lib/site-packages/dill/detect.py b/venv/Lib/site-packages/dill/detect.py
new file mode 100644
index 0000000000000000000000000000000000000000..cdd428765dfbe427e3423e3a8294260a47b2751d
--- /dev/null
+++ b/venv/Lib/site-packages/dill/detect.py
@@ -0,0 +1,287 @@
+#!/usr/bin/env python
+#
+# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
+# Copyright (c) 2008-2016 California Institute of Technology.
+# Copyright (c) 2016-2026 The Uncertainty Quantification Foundation.
+# License: 3-clause BSD. The full license text is available at:
+# - https://github.com/uqfoundation/dill/blob/master/LICENSE
+"""
+Methods for detecting objects leading to pickling failures.
+"""
+
+import dis
+from inspect import ismethod, isfunction, istraceback, isframe, iscode
+
+from .pointers import parent, reference, at, parents, children
+from .logger import trace
+
+__all__ = ['baditems','badobjects','badtypes','code','errors','freevars',
+ 'getmodule','globalvars','nestedcode','nestedglobals','outermost',
+ 'referredglobals','referrednested','trace','varnames']
+
+def getmodule(object, _filename=None, force=False):
+ """get the module of the object"""
+ from inspect import getmodule as getmod
+ module = getmod(object, _filename)
+ if module or not force: return module
+ import builtins
+ from .source import getname
+ name = getname(object, force=True)
+ return builtins if name in vars(builtins).keys() else None
+
+def outermost(func): # is analogous to getsource(func,enclosing=True)
+ """get outermost enclosing object (i.e. the outer function in a closure)
+
+ NOTE: this is the object-equivalent of getsource(func, enclosing=True)
+ """
+ if ismethod(func):
+ _globals = func.__func__.__globals__ or {}
+ elif isfunction(func):
+ _globals = func.__globals__ or {}
+ else:
+ return #XXX: or raise? no matches
+ _globals = _globals.items()
+ # get the enclosing source
+ from .source import getsourcelines
+ try: lines,lnum = getsourcelines(func, enclosing=True)
+ except Exception: #TypeError, IOError
+ lines,lnum = [],None
+ code = ''.join(lines)
+ # get all possible names,objects that are named in the enclosing source
+ _locals = ((name,obj) for (name,obj) in _globals if name in code)
+ # now only save the objects that generate the enclosing block
+ for name,obj in _locals: #XXX: don't really need 'name'
+ try:
+ if getsourcelines(obj) == (lines,lnum): return obj
+ except Exception: #TypeError, IOError
+ pass
+ return #XXX: or raise? no matches
+
+def nestedcode(func, recurse=True): #XXX: or return dict of {co_name: co} ?
+ """get the code objects for any nested functions (e.g. in a closure)"""
+ func = code(func)
+ if not iscode(func): return [] #XXX: or raise? no matches
+ nested = set()
+ for co in func.co_consts:
+ if co is None: continue
+ co = code(co)
+ if co:
+ nested.add(co)
+ if recurse: nested |= set(nestedcode(co, recurse=True))
+ return list(nested)
+
+def code(func):
+ """get the code object for the given function or method
+
+ NOTE: use dill.source.getsource(CODEOBJ) to get the source code
+ """
+ if ismethod(func): func = func.__func__
+ if isfunction(func): func = func.__code__
+ if istraceback(func): func = func.tb_frame
+ if isframe(func): func = func.f_code
+ if iscode(func): return func
+ return
+
+#XXX: ugly: parse dis.dis for name after " len(referrednested(func)), try calling func().
+ If possible, python builds code objects, but delays building functions
+ until func() is called.
+ """
+ import gc
+ funcs = set()
+ # get the code objects, and try to track down by referrence
+ for co in nestedcode(func, recurse):
+ # look for function objects that refer to the code object
+ for obj in gc.get_referrers(co):
+ # get methods
+ _ = getattr(obj, '__func__', None) # ismethod
+ if getattr(_, '__code__', None) is co: funcs.add(obj)
+ # get functions
+ elif getattr(obj, '__code__', None) is co: funcs.add(obj)
+ # get frame objects
+ elif getattr(obj, 'f_code', None) is co: funcs.add(obj)
+ # get code objects
+ elif hasattr(obj, 'co_code') and obj is co: funcs.add(obj)
+# frameobjs => func.__code__.co_varnames not in func.__code__.co_cellvars
+# funcobjs => func.__code__.co_cellvars not in func.__code__.co_varnames
+# frameobjs are not found, however funcobjs are...
+# (see: test_mixins.quad ... and test_mixins.wtf)
+# after execution, code objects get compiled, and then may be found by gc
+ return list(funcs)
+
+
+def freevars(func):
+ """get objects defined in enclosing code that are referred to by func
+
+ returns a dict of {name:object}"""
+ if ismethod(func): func = func.__func__
+ if isfunction(func):
+ closures = func.__closure__ or ()
+ func = func.__code__.co_freevars # get freevars
+ else:
+ return {}
+
+ def get_cell_contents():
+ for name, c in zip(func, closures):
+ try:
+ cell_contents = c.cell_contents
+ except ValueError: # cell is empty
+ continue
+ yield name, c.cell_contents
+
+ return dict(get_cell_contents())
+
+# thanks to Davies Liu for recursion of globals
+def nestedglobals(func, recurse=True):
+ """get the names of any globals found within func"""
+ func = code(func)
+ if func is None: return list()
+ import sys
+ from .temp import capture
+ CAN_NULL = sys.hexversion >= 0x30b00a7 # NULL may be prepended >= 3.11a7
+ names = set()
+ with capture('stdout') as out:
+ try:
+ dis.dis(func) #XXX: dis.dis(None) disassembles last traceback
+ except IndexError:
+ pass #FIXME: HACK for IS_PYPY (3.11)
+ for line in out.getvalue().splitlines():
+ if '_GLOBAL' in line:
+ name = line.split('(')[-1].split(')')[0]
+ if CAN_NULL:
+ names.add(name.replace('NULL + ', '').replace(' + NULL', ''))
+ else:
+ names.add(name)
+ for co in getattr(func, 'co_consts', tuple()):
+ if co and recurse and iscode(co):
+ names.update(nestedglobals(co, recurse=True))
+ return list(names)
+
+def referredglobals(func, recurse=True, builtin=False):
+ """get the names of objects in the global scope referred to by func"""
+ return globalvars(func, recurse, builtin).keys()
+
+def globalvars(func, recurse=True, builtin=False):
+ """get objects defined in global scope that are referred to by func
+
+ return a dict of {name:object}"""
+ if ismethod(func): func = func.__func__
+ if isfunction(func):
+ globs = vars(getmodule(sum)).copy() if builtin else {}
+ # get references from within closure
+ orig_func, func = func, set()
+ for obj in orig_func.__closure__ or {}:
+ try:
+ cell_contents = obj.cell_contents
+ except ValueError: # cell is empty
+ pass
+ else:
+ _vars = globalvars(cell_contents, recurse, builtin) or {}
+ func.update(_vars) #XXX: (above) be wary of infinte recursion?
+ globs.update(_vars)
+ # get globals
+ globs.update(orig_func.__globals__ or {})
+ # get names of references
+ if not recurse:
+ func.update(orig_func.__code__.co_names)
+ else:
+ func.update(nestedglobals(orig_func.__code__))
+ # find globals for all entries of func
+ for key in func.copy(): #XXX: unnecessary...?
+ nested_func = globs.get(key)
+ if nested_func is orig_func:
+ #func.remove(key) if key in func else None
+ continue #XXX: globalvars(func, False)?
+ func.update(globalvars(nested_func, True, builtin))
+ elif iscode(func):
+ globs = vars(getmodule(sum)).copy() if builtin else {}
+ #globs.update(globals())
+ if not recurse:
+ func = func.co_names # get names
+ else:
+ orig_func = func.co_name # to stop infinite recursion
+ func = set(nestedglobals(func))
+ # find globals for all entries of func
+ for key in func.copy(): #XXX: unnecessary...?
+ if key is orig_func:
+ #func.remove(key) if key in func else None
+ continue #XXX: globalvars(func, False)?
+ nested_func = globs.get(key)
+ func.update(globalvars(nested_func, True, builtin))
+ else:
+ return {}
+ #NOTE: if name not in __globals__, then we skip it...
+ return dict((name,globs[name]) for name in func if name in globs)
+
+
+def varnames(func):
+ """get names of variables defined by func
+
+ returns a tuple (local vars, local vars referrenced by nested functions)"""
+ func = code(func)
+ if not iscode(func):
+ return () #XXX: better ((),())? or None?
+ return func.co_varnames, func.co_cellvars
+
+
+def baditems(obj, exact=False, safe=False): #XXX: obj=globals() ?
+ """get items in object that fail to pickle"""
+ if not hasattr(obj,'__iter__'): # is not iterable
+ return [j for j in (badobjects(obj,0,exact,safe),) if j is not None]
+ obj = obj.values() if getattr(obj,'values',None) else obj
+ _obj = [] # can't use a set, as items may be unhashable
+ [_obj.append(badobjects(i,0,exact,safe)) for i in obj if i not in _obj]
+ return [j for j in _obj if j is not None]
+
+
+def badobjects(obj, depth=0, exact=False, safe=False):
+ """get objects that fail to pickle"""
+ from dill import pickles
+ if not depth:
+ if pickles(obj,exact,safe): return None
+ return obj
+ return dict(((attr, badobjects(getattr(obj,attr),depth-1,exact,safe)) \
+ for attr in dir(obj) if not pickles(getattr(obj,attr),exact,safe)))
+
+def badtypes(obj, depth=0, exact=False, safe=False):
+ """get types for objects that fail to pickle"""
+ from dill import pickles
+ if not depth:
+ if pickles(obj,exact,safe): return None
+ return type(obj)
+ return dict(((attr, badtypes(getattr(obj,attr),depth-1,exact,safe)) \
+ for attr in dir(obj) if not pickles(getattr(obj,attr),exact,safe)))
+
+def errors(obj, depth=0, exact=False, safe=False):
+ """get errors for objects that fail to pickle"""
+ from dill import pickles, copy
+ if not depth:
+ try:
+ pik = copy(obj)
+ if exact:
+ assert pik == obj, \
+ "Unpickling produces %s instead of %s" % (pik,obj)
+ assert type(pik) == type(obj), \
+ "Unpickling produces %s instead of %s" % (type(pik),type(obj))
+ return None
+ except Exception:
+ import sys
+ return sys.exc_info()[1]
+ _dict = {}
+ for attr in dir(obj):
+ try:
+ _attr = getattr(obj,attr)
+ except Exception:
+ import sys
+ _dict[attr] = sys.exc_info()[1]
+ continue
+ if not pickles(_attr,exact,safe):
+ _dict[attr] = errors(_attr,depth-1,exact,safe)
+ return _dict
+
+
+# EOF
diff --git a/venv/Lib/site-packages/dill/logger.py b/venv/Lib/site-packages/dill/logger.py
new file mode 100644
index 0000000000000000000000000000000000000000..a33011227beaf8f6b8d11ec7b4c2247fcb6ef143
--- /dev/null
+++ b/venv/Lib/site-packages/dill/logger.py
@@ -0,0 +1,285 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Author: Leonardo Gama (@leogama)
+# Copyright (c) 2022-2026 The Uncertainty Quantification Foundation.
+# License: 3-clause BSD. The full license text is available at:
+# - https://github.com/uqfoundation/dill/blob/master/LICENSE
+"""
+Logging utilities for dill.
+
+The 'logger' object is dill's top-level logger.
+
+The 'adapter' object wraps the logger and implements a 'trace()' method that
+generates a detailed tree-style trace for the pickling call at log level INFO.
+
+The 'trace()' function sets and resets dill's logger log level, enabling and
+disabling the pickling trace.
+
+The trace shows a tree structure depicting the depth of each object serialized
+*with dill save functions*, but not the ones that use save functions from
+'pickle._Pickler.dispatch'. If the information is available, it also displays
+the size in bytes that the object contributed to the pickle stream (including
+its child objects). Sample trace output:
+
+ >>> import dill, dill.tests
+ >>> dill.detect.trace(True)
+ >>> dill.dump_session(main=dill.tests)
+ ┬ M1:
+ ├┬ F2:
+ │└ # F2 [32 B]
+ ├┬ D2:
+ │├┬ T4:
+ ││└ # T4 [35 B]
+ │├┬ D2:
+ ││├┬ T4:
+ │││└ # T4 [50 B]
+ ││├┬ D2:
+ │││└ # D2 [84 B]
+ ││└ # D2 [413 B]
+ │└ # D2 [763 B]
+ └ # M1 [813 B]
+"""
+
+__all__ = ['adapter', 'logger', 'trace']
+
+import codecs
+import contextlib
+import locale
+import logging
+import math
+import os
+from functools import partial
+from typing import TextIO, Union
+
+import dill
+
+# Tree drawing characters: Unicode to ASCII map.
+ASCII_MAP = str.maketrans({"│": "|", "├": "|", "┬": "+", "└": "`"})
+
+## Notes about the design choices ##
+
+# Here is some domumentation of the Standard Library's logging internals that
+# can't be found completely in the official documentation. dill's logger is
+# obtained by calling logging.getLogger('dill') and therefore is an instance of
+# logging.getLoggerClass() at the call time. As this is controlled by the user,
+# in order to add some functionality to it it's necessary to use a LoggerAdapter
+# to wrap it, overriding some of the adapter's methods and creating new ones.
+#
+# Basic calling sequence
+# ======================
+#
+# Python's logging functionality can be conceptually divided into five steps:
+# 0. Check logging level -> abort if call level is greater than logger level
+# 1. Gather information -> construct a LogRecord from passed arguments and context
+# 2. Filter (optional) -> discard message if the record matches a filter
+# 3. Format -> format message with args, then format output string with message plus record
+# 4. Handle -> write the formatted string to output as defined in the handler
+#
+# dill.logging.logger.log -> # or logger.info, etc.
+# Logger.log -> \
+# Logger._log -> }- accept 'extra' parameter for custom record entries
+# Logger.makeRecord -> /
+# LogRecord.__init__
+# Logger.handle ->
+# Logger.callHandlers ->
+# Handler.handle ->
+# Filterer.filter ->
+# Filter.filter
+# StreamHandler.emit ->
+# Handler.format ->
+# Formatter.format ->
+# LogRecord.getMessage # does: record.message = msg % args
+# Formatter.formatMessage ->
+# PercentStyle.format # does: self._fmt % vars(record)
+#
+# NOTE: All methods from the second line on are from logging.__init__.py
+
+class TraceAdapter(logging.LoggerAdapter):
+ """
+ Tracks object tree depth and calculates pickled object size.
+
+ A single instance of this wraps the module's logger, as the logging API
+ doesn't allow setting it directly with a custom Logger subclass. The added
+ 'trace()' method receives a pickle instance as the first argument and
+ creates extra values to be added in the LogRecord from it, then calls
+ 'info()'.
+
+ Usage of logger with 'trace()' method:
+
+ >>> from dill.logger import adapter as logger #NOTE: not dill.logger.logger
+ >>> ...
+ >>> def save_atype(pickler, obj):
+ >>> logger.trace(pickler, "Message with %s and %r etc. placeholders", 'text', obj)
+ >>> ...
+ """
+ def __init__(self, logger):
+ self.logger = logger
+ def addHandler(self, handler):
+ formatter = TraceFormatter("%(prefix)s%(message)s%(suffix)s", handler=handler)
+ handler.setFormatter(formatter)
+ self.logger.addHandler(handler)
+ def removeHandler(self, handler):
+ self.logger.removeHandler(handler)
+ def process(self, msg, kwargs):
+ # A no-op override, as we don't have self.extra.
+ return msg, kwargs
+ def trace_setup(self, pickler):
+ # Called by Pickler.dump().
+ if not dill._dill.is_dill(pickler, child=False):
+ return
+ if self.isEnabledFor(logging.INFO):
+ pickler._trace_depth = 1
+ pickler._size_stack = []
+ else:
+ pickler._trace_depth = None
+ def trace(self, pickler, msg, *args, **kwargs):
+ if not hasattr(pickler, '_trace_depth'):
+ logger.info(msg, *args, **kwargs)
+ return
+ if pickler._trace_depth is None:
+ return
+ extra = kwargs.get('extra', {})
+ pushed_obj = msg.startswith('#')
+ size = None
+ try:
+ # Streams are not required to be tellable.
+ size = pickler._file.tell()
+ frame = pickler.framer.current_frame
+ try:
+ size += frame.tell()
+ except AttributeError:
+ # PyPy may use a BytesBuilder as frame
+ size += len(frame)
+ except (AttributeError, TypeError):
+ pass
+ if size is not None:
+ if not pushed_obj:
+ pickler._size_stack.append(size)
+ else:
+ size -= pickler._size_stack.pop()
+ extra['size'] = size
+ if pushed_obj:
+ pickler._trace_depth -= 1
+ extra['depth'] = pickler._trace_depth
+ kwargs['extra'] = extra
+ self.info(msg, *args, **kwargs)
+ if not pushed_obj:
+ pickler._trace_depth += 1
+
+class TraceFormatter(logging.Formatter):
+ """
+ Generates message prefix and suffix from record.
+
+ This Formatter adds prefix and suffix strings to the log message in trace
+ mode (an also provides empty string defaults for normal logs).
+ """
+ def __init__(self, *args, handler=None, **kwargs):
+ super().__init__(*args, **kwargs)
+ try:
+ encoding = handler.stream.encoding
+ if encoding is None:
+ raise AttributeError
+ except AttributeError:
+ encoding = locale.getpreferredencoding()
+ try:
+ encoding = codecs.lookup(encoding).name
+ except LookupError:
+ self.is_utf8 = False
+ else:
+ self.is_utf8 = (encoding == codecs.lookup('utf-8').name)
+ def format(self, record):
+ fields = {'prefix': "", 'suffix': ""}
+ if getattr(record, 'depth', 0) > 0:
+ if record.msg.startswith("#"):
+ prefix = (record.depth - 1)*"│" + "└"
+ elif record.depth == 1:
+ prefix = "┬"
+ else:
+ prefix = (record.depth - 2)*"│" + "├┬"
+ if not self.is_utf8:
+ prefix = prefix.translate(ASCII_MAP) + "-"
+ fields['prefix'] = prefix + " "
+ if hasattr(record, 'size') and record.size is not None and record.size >= 1:
+ # Show object size in human-readable form.
+ power = int(math.log(record.size, 2)) // 10
+ size = record.size >> power*10
+ fields['suffix'] = " [%d %sB]" % (size, "KMGTP"[power] + "i" if power else "")
+ vars(record).update(fields)
+ return super().format(record)
+
+logger = logging.getLogger('dill')
+logger.propagate = False
+adapter = TraceAdapter(logger)
+stderr_handler = logging._StderrHandler()
+adapter.addHandler(stderr_handler)
+
+def trace(arg: Union[bool, TextIO, str, os.PathLike] = None, *, mode: str = 'a') -> None:
+ """print a trace through the stack when pickling; useful for debugging
+
+ With a single boolean argument, enable or disable the tracing.
+
+ Example usage:
+
+ >>> import dill
+ >>> dill.detect.trace(True)
+ >>> dill.dump_session()
+
+ Alternatively, ``trace()`` can be used as a context manager. With no
+ arguments, it just takes care of restoring the tracing state on exit.
+ Either a file handle, or a file name and (optionally) a file mode may be
+ specitfied to redirect the tracing output in the ``with`` block context. A
+ log function is yielded by the manager so the user can write extra
+ information to the file.
+
+ Example usage:
+
+ >>> from dill import detect
+ >>> D = {'a': 42, 'b': {'x': None}}
+ >>> with detect.trace():
+ >>> dumps(D)
+ ┬ D2:
+ ├┬ D2:
+ │└ # D2 [8 B]
+ └ # D2 [22 B]
+ >>> squared = lambda x: x**2
+ >>> with detect.trace('output.txt', mode='w') as log:
+ >>> log("> D = %r", D)
+ >>> dumps(D)
+ >>> log("> squared = %r", squared)
+ >>> dumps(squared)
+
+ Arguments:
+ arg: a boolean value, or an optional file-like or path-like object for the context manager
+ mode: mode string for ``open()`` if a file name is passed as the first argument
+ """
+ if repr(arg) not in ('False', 'True'):
+ return TraceManager(file=arg, mode=mode)
+ logger.setLevel(logging.INFO if arg else logging.WARNING)
+
+class TraceManager(contextlib.AbstractContextManager):
+ """context manager version of trace(); can redirect the trace to a file"""
+ def __init__(self, file, mode):
+ self.file = file
+ self.mode = mode
+ self.redirect = file is not None
+ self.file_is_stream = hasattr(file, 'write')
+ def __enter__(self):
+ if self.redirect:
+ stderr_handler.flush()
+ if self.file_is_stream:
+ self.handler = logging.StreamHandler(self.file)
+ else:
+ self.handler = logging.FileHandler(self.file, self.mode)
+ adapter.removeHandler(stderr_handler)
+ adapter.addHandler(self.handler)
+ self.old_level = adapter.getEffectiveLevel()
+ adapter.setLevel(logging.INFO)
+ return adapter.info
+ def __exit__(self, *exc_info):
+ adapter.setLevel(self.old_level)
+ if self.redirect:
+ adapter.removeHandler(self.handler)
+ adapter.addHandler(stderr_handler)
+ if not self.file_is_stream:
+ self.handler.close()
diff --git a/venv/Lib/site-packages/dill/objtypes.py b/venv/Lib/site-packages/dill/objtypes.py
new file mode 100644
index 0000000000000000000000000000000000000000..bf88a47fdd07d3c6cbae71e8599e7f3e2180d442
--- /dev/null
+++ b/venv/Lib/site-packages/dill/objtypes.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+#
+# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
+# Copyright (c) 2008-2016 California Institute of Technology.
+# Copyright (c) 2016-2026 The Uncertainty Quantification Foundation.
+# License: 3-clause BSD. The full license text is available at:
+# - https://github.com/uqfoundation/dill/blob/master/LICENSE
+"""
+all Python Standard Library object types (currently: CH 1-15 @ 2.7)
+and some other common object types (i.e. numpy.ndarray)
+
+to load more objects and types, use dill.load_types()
+"""
+
+# non-local import of dill.objects
+from dill import objects
+for _type in objects.keys():
+ exec("%s = type(objects['%s'])" % (_type,_type))
+
+del objects
+try:
+ del _type
+except NameError:
+ pass
diff --git a/venv/Lib/site-packages/dill/pointers.py b/venv/Lib/site-packages/dill/pointers.py
new file mode 100644
index 0000000000000000000000000000000000000000..d27745febe7dfa357e295b9f4b8dcaedcb4f3496
--- /dev/null
+++ b/venv/Lib/site-packages/dill/pointers.py
@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+#
+# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
+# Copyright (c) 2008-2016 California Institute of Technology.
+# Copyright (c) 2016-2026 The Uncertainty Quantification Foundation.
+# License: 3-clause BSD. The full license text is available at:
+# - https://github.com/uqfoundation/dill/blob/master/LICENSE
+
+__all__ = ['parent', 'reference', 'at', 'parents', 'children']
+
+import gc
+import sys
+
+from ._dill import _proxy_helper as reference
+from ._dill import _locate_object as at
+
+def parent(obj, objtype, ignore=()):
+ """
+>>> listiter = iter([4,5,6,7])
+>>> obj = parent(listiter, list)
+>>> obj == [4,5,6,7] # actually 'is', but don't have handle any longer
+True
+
+NOTE: objtype can be a single type (e.g. int or list) or a tuple of types.
+
+WARNING: if obj is a sequence (e.g. list), may produce unexpected results.
+Parent finds *one* parent (e.g. the last member of the sequence).
+ """
+ depth = 1 #XXX: always looking for the parent (only, right?)
+ chain = parents(obj, objtype, depth, ignore)
+ parent = chain.pop()
+ if parent is obj:
+ return None
+ return parent
+
+
+def parents(obj, objtype, depth=1, ignore=()): #XXX: objtype=object ?
+ """Find the chain of referents for obj. Chain will end with obj.
+
+ objtype: an object type or tuple of types to search for
+ depth: search depth (e.g. depth=2 is 'grandparents')
+ ignore: an object or tuple of objects to ignore in the search
+ """
+ edge_func = gc.get_referents # looking for refs, not back_refs
+ predicate = lambda x: isinstance(x, objtype) # looking for parent type
+ #if objtype is None: predicate = lambda x: True #XXX: in obj.mro() ?
+ ignore = (ignore,) if not hasattr(ignore, '__len__') else ignore
+ ignore = (id(obj) for obj in ignore)
+ chain = find_chain(obj, predicate, edge_func, depth)[::-1]
+ #XXX: should pop off obj... ?
+ return chain
+
+
+def children(obj, objtype, depth=1, ignore=()): #XXX: objtype=object ?
+ """Find the chain of referrers for obj. Chain will start with obj.
+
+ objtype: an object type or tuple of types to search for
+ depth: search depth (e.g. depth=2 is 'grandchildren')
+ ignore: an object or tuple of objects to ignore in the search
+
+ NOTE: a common thing to ignore is all globals, 'ignore=(globals(),)'
+
+ NOTE: repeated calls may yield different results, as python stores
+ the last value in the special variable '_'; thus, it is often good
+ to execute something to replace '_' (e.g. >>> 1+1).
+ """
+ edge_func = gc.get_referrers # looking for back_refs, not refs
+ predicate = lambda x: isinstance(x, objtype) # looking for child type
+ #if objtype is None: predicate = lambda x: True #XXX: in obj.mro() ?
+ ignore = (ignore,) if not hasattr(ignore, '__len__') else ignore
+ ignore = (id(obj) for obj in ignore)
+ chain = find_chain(obj, predicate, edge_func, depth, ignore)
+ #XXX: should pop off obj... ?
+ return chain
+
+
+# more generic helper function (cut-n-paste from objgraph)
+# Source at http://mg.pov.lt/objgraph/
+# Copyright (c) 2008-2010 Marius Gedminas
+# Copyright (c) 2010 Stefano Rivera
+# Released under the MIT licence (see objgraph/objgrah.py)
+
+def find_chain(obj, predicate, edge_func, max_depth=20, extra_ignore=()):
+ queue = [obj]
+ depth = {id(obj): 0}
+ parent = {id(obj): None}
+ ignore = set(extra_ignore)
+ ignore.add(id(extra_ignore))
+ ignore.add(id(queue))
+ ignore.add(id(depth))
+ ignore.add(id(parent))
+ ignore.add(id(ignore))
+ ignore.add(id(sys._getframe())) # this function
+ ignore.add(id(sys._getframe(1))) # find_chain/find_backref_chain, likely
+ gc.collect()
+ while queue:
+ target = queue.pop(0)
+ if predicate(target):
+ chain = [target]
+ while parent[id(target)] is not None:
+ target = parent[id(target)]
+ chain.append(target)
+ return chain
+ tdepth = depth[id(target)]
+ if tdepth < max_depth:
+ referrers = edge_func(target)
+ ignore.add(id(referrers))
+ for source in referrers:
+ if id(source) in ignore:
+ continue
+ if id(source) not in depth:
+ depth[id(source)] = tdepth + 1
+ parent[id(source)] = target
+ queue.append(source)
+ return [obj] # not found
+
+
+# backward compatibility
+refobject = at
+
+
+# EOF
diff --git a/venv/Lib/site-packages/dill/session.py b/venv/Lib/site-packages/dill/session.py
new file mode 100644
index 0000000000000000000000000000000000000000..b5dd2ff8d4b3b1af84df9f07fba731be4011712f
--- /dev/null
+++ b/venv/Lib/site-packages/dill/session.py
@@ -0,0 +1,612 @@
+#!/usr/bin/env python
+#
+# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
+# Author: Leonardo Gama (@leogama)
+# Copyright (c) 2008-2015 California Institute of Technology.
+# Copyright (c) 2016-2026 The Uncertainty Quantification Foundation.
+# License: 3-clause BSD. The full license text is available at:
+# - https://github.com/uqfoundation/dill/blob/master/LICENSE
+"""
+Pickle and restore the intepreter session.
+"""
+
+__all__ = [
+ 'dump_module', 'load_module', 'load_module_asdict',
+ 'dump_session', 'load_session' # backward compatibility
+]
+
+import re
+import os
+import sys
+import warnings
+import pathlib
+import tempfile
+
+TEMPDIR = pathlib.PurePath(tempfile.gettempdir())
+
+# Type hints.
+from typing import Optional, Union
+
+from dill import _dill, Pickler, Unpickler
+from ._dill import (
+ BuiltinMethodType, FunctionType, MethodType, ModuleType, TypeType,
+ _import_module, _is_builtin_module, _is_imported_module, _main_module,
+ _reverse_typemap, __builtin__, UnpicklingError,
+)
+
+def _module_map():
+ """get map of imported modules"""
+ from collections import defaultdict
+ from types import SimpleNamespace
+ modmap = SimpleNamespace(
+ by_name=defaultdict(list),
+ by_id=defaultdict(list),
+ top_level={},
+ )
+ for modname, module in sys.modules.items():
+ if modname in ('__main__', '__mp_main__') or not isinstance(module, ModuleType):
+ continue
+ if '.' not in modname:
+ modmap.top_level[id(module)] = modname
+ for objname, modobj in module.__dict__.items():
+ modmap.by_name[objname].append((modobj, modname))
+ modmap.by_id[id(modobj)].append((modobj, objname, modname))
+ return modmap
+
+IMPORTED_AS_TYPES = (ModuleType, TypeType, FunctionType, MethodType, BuiltinMethodType)
+if 'PyCapsuleType' in _reverse_typemap:
+ IMPORTED_AS_TYPES += (_reverse_typemap['PyCapsuleType'],)
+IMPORTED_AS_MODULES = ('ctypes', 'typing', 'subprocess', 'threading',
+ r'concurrent\.futures(\.\w+)?', r'multiprocessing(\.\w+)?')
+IMPORTED_AS_MODULES = tuple(re.compile(x) for x in IMPORTED_AS_MODULES)
+
+def _lookup_module(modmap, name, obj, main_module):
+ """lookup name or id of obj if module is imported"""
+ for modobj, modname in modmap.by_name[name]:
+ if modobj is obj and sys.modules[modname] is not main_module:
+ return modname, name
+ __module__ = getattr(obj, '__module__', None)
+ if isinstance(obj, IMPORTED_AS_TYPES) or (__module__ is not None
+ and any(regex.fullmatch(__module__) for regex in IMPORTED_AS_MODULES)):
+ for modobj, objname, modname in modmap.by_id[id(obj)]:
+ if sys.modules[modname] is not main_module:
+ return modname, objname
+ return None, None
+
+def _stash_modules(main_module):
+ modmap = _module_map()
+ newmod = ModuleType(main_module.__name__)
+
+ imported = []
+ imported_as = []
+ imported_top_level = [] # keep separated for backward compatibility
+ original = {}
+ for name, obj in main_module.__dict__.items():
+ if obj is main_module:
+ original[name] = newmod # self-reference
+ elif obj is main_module.__dict__:
+ original[name] = newmod.__dict__
+ # Avoid incorrectly matching a singleton value in another package (ex.: __doc__).
+ elif any(obj is singleton for singleton in (None, False, True)) \
+ or isinstance(obj, ModuleType) and _is_builtin_module(obj): # always saved by ref
+ original[name] = obj
+ else:
+ source_module, objname = _lookup_module(modmap, name, obj, main_module)
+ if source_module is not None:
+ if objname == name:
+ imported.append((source_module, name))
+ else:
+ imported_as.append((source_module, objname, name))
+ else:
+ try:
+ imported_top_level.append((modmap.top_level[id(obj)], name))
+ except KeyError:
+ original[name] = obj
+
+ if len(original) < len(main_module.__dict__):
+ newmod.__dict__.update(original)
+ newmod.__dill_imported = imported
+ newmod.__dill_imported_as = imported_as
+ newmod.__dill_imported_top_level = imported_top_level
+ if getattr(newmod, '__loader__', None) is None and _is_imported_module(main_module):
+ # Trick _is_imported_module() to force saving as an imported module.
+ newmod.__loader__ = True # will be discarded by save_module()
+ return newmod
+ else:
+ return main_module
+
+def _restore_modules(unpickler, main_module):
+ try:
+ for modname, name in main_module.__dict__.pop('__dill_imported'):
+ main_module.__dict__[name] = unpickler.find_class(modname, name)
+ for modname, objname, name in main_module.__dict__.pop('__dill_imported_as'):
+ main_module.__dict__[name] = unpickler.find_class(modname, objname)
+ for modname, name in main_module.__dict__.pop('__dill_imported_top_level'):
+ main_module.__dict__[name] = __import__(modname)
+ except KeyError:
+ pass
+
+#NOTE: 06/03/15 renamed main_module to main
+def dump_module(
+ filename: Union[str, os.PathLike] = None,
+ module: Optional[Union[ModuleType, str]] = None,
+ refimported: bool = False,
+ **kwds
+) -> None:
+ """Pickle the current state of :py:mod:`__main__` or another module to a file.
+
+ Save the contents of :py:mod:`__main__` (e.g. from an interactive
+ interpreter session), an imported module, or a module-type object (e.g.
+ built with :py:class:`~types.ModuleType`), to a file. The pickled
+ module can then be restored with the function :py:func:`load_module`.
+
+ Args:
+ filename: a path-like object or a writable stream. If `None`
+ (the default), write to a named file in a temporary directory.
+ module: a module object or the name of an importable module. If `None`
+ (the default), :py:mod:`__main__` is saved.
+ refimported: if `True`, all objects identified as having been imported
+ into the module's namespace are saved by reference. *Note:* this is
+ similar but independent from ``dill.settings[`byref`]``, as
+ ``refimported`` refers to virtually all imported objects, while
+ ``byref`` only affects select objects.
+ **kwds: extra keyword arguments passed to :py:class:`Pickler()`.
+
+ Raises:
+ :py:exc:`PicklingError`: if pickling fails.
+
+ Examples:
+
+ - Save current interpreter session state:
+
+ >>> import dill
+ >>> squared = lambda x: x*x
+ >>> dill.dump_module() # save state of __main__ to /tmp/session.pkl
+
+ - Save the state of an imported/importable module:
+
+ >>> import dill
+ >>> import pox
+ >>> pox.plus_one = lambda x: x+1
+ >>> dill.dump_module('pox_session.pkl', module=pox)
+
+ - Save the state of a non-importable, module-type object:
+
+ >>> import dill
+ >>> from types import ModuleType
+ >>> foo = ModuleType('foo')
+ >>> foo.values = [1,2,3]
+ >>> import math
+ >>> foo.sin = math.sin
+ >>> dill.dump_module('foo_session.pkl', module=foo, refimported=True)
+
+ - Restore the state of the saved modules:
+
+ >>> import dill
+ >>> dill.load_module()
+ >>> squared(2)
+ 4
+ >>> pox = dill.load_module('pox_session.pkl')
+ >>> pox.plus_one(1)
+ 2
+ >>> foo = dill.load_module('foo_session.pkl')
+ >>> [foo.sin(x) for x in foo.values]
+ [0.8414709848078965, 0.9092974268256817, 0.1411200080598672]
+
+ - Use `refimported` to save imported objects by reference:
+
+ >>> import dill
+ >>> from html.entities import html5
+ >>> type(html5), len(html5)
+ (dict, 2231)
+ >>> import io
+ >>> buf = io.BytesIO()
+ >>> dill.dump_module(buf) # saves __main__, with html5 saved by value
+ >>> len(buf.getvalue()) # pickle size in bytes
+ 71665
+ >>> buf = io.BytesIO()
+ >>> dill.dump_module(buf, refimported=True) # html5 saved by reference
+ >>> len(buf.getvalue())
+ 438
+
+ *Changed in version 0.3.6:* Function ``dump_session()`` was renamed to
+ ``dump_module()``. Parameters ``main`` and ``byref`` were renamed to
+ ``module`` and ``refimported``, respectively.
+
+ Note:
+ Currently, ``dill.settings['byref']`` and ``dill.settings['recurse']``
+ don't apply to this function.
+ """
+ for old_par, par in [('main', 'module'), ('byref', 'refimported')]:
+ if old_par in kwds:
+ message = "The argument %r has been renamed %r" % (old_par, par)
+ if old_par == 'byref':
+ message += " to distinguish it from dill.settings['byref']"
+ warnings.warn(message + ".", PendingDeprecationWarning)
+ if locals()[par]: # the defaults are None and False
+ raise TypeError("both %r and %r arguments were used" % (par, old_par))
+ refimported = kwds.pop('byref', refimported)
+ module = kwds.pop('main', module)
+
+ from .settings import settings
+ protocol = settings['protocol']
+ main = module
+ if main is None:
+ main = _main_module
+ elif isinstance(main, str):
+ main = _import_module(main)
+ if not isinstance(main, ModuleType):
+ raise TypeError("%r is not a module" % main)
+ if hasattr(filename, 'write'):
+ file = filename
+ else:
+ if filename is None:
+ filename = str(TEMPDIR/'session.pkl')
+ file = open(filename, 'wb')
+ try:
+ pickler = Pickler(file, protocol, **kwds)
+ pickler._original_main = main
+ if refimported:
+ main = _stash_modules(main)
+ pickler._main = main #FIXME: dill.settings are disabled
+ pickler._byref = False # disable pickling by name reference
+ pickler._recurse = False # disable pickling recursion for globals
+ pickler._session = True # is best indicator of when pickling a session
+ pickler._first_pass = True
+ pickler._main_modified = main is not pickler._original_main
+ pickler.dump(main)
+ finally:
+ if file is not filename: # if newly opened file
+ file.close()
+ return
+
+# Backward compatibility.
+def dump_session(filename=None, main=None, byref=False, **kwds):
+ warnings.warn("dump_session() has been renamed dump_module()", PendingDeprecationWarning)
+ dump_module(filename, module=main, refimported=byref, **kwds)
+dump_session.__doc__ = dump_module.__doc__
+
+class _PeekableReader:
+ """lightweight stream wrapper that implements peek()"""
+ def __init__(self, stream):
+ self.stream = stream
+ def read(self, n):
+ return self.stream.read(n)
+ def readline(self):
+ return self.stream.readline()
+ def tell(self):
+ return self.stream.tell()
+ def close(self):
+ return self.stream.close()
+ def peek(self, n):
+ stream = self.stream
+ try:
+ if hasattr(stream, 'flush'): stream.flush()
+ position = stream.tell()
+ stream.seek(position) # assert seek() works before reading
+ chunk = stream.read(n)
+ stream.seek(position)
+ return chunk
+ except (AttributeError, OSError):
+ raise NotImplementedError("stream is not peekable: %r", stream) from None
+
+def _make_peekable(stream):
+ """return stream as an object with a peek() method"""
+ import io
+ if hasattr(stream, 'peek'):
+ return stream
+ if not (hasattr(stream, 'tell') and hasattr(stream, 'seek')):
+ try:
+ return io.BufferedReader(stream)
+ except Exception:
+ pass
+ return _PeekableReader(stream)
+
+def _identify_module(file, main=None):
+ """identify the name of the module stored in the given file-type object"""
+ from pickletools import genops
+ UNICODE = {'UNICODE', 'BINUNICODE', 'SHORT_BINUNICODE'}
+ found_import = False
+ try:
+ for opcode, arg, pos in genops(file.peek(256)):
+ if not found_import:
+ if opcode.name in ('GLOBAL', 'SHORT_BINUNICODE') and \
+ arg.endswith('_import_module'):
+ found_import = True
+ else:
+ if opcode.name in UNICODE:
+ return arg
+ else:
+ raise UnpicklingError("reached STOP without finding main module")
+ except (NotImplementedError, ValueError) as error:
+ # ValueError occours when the end of the chunk is reached (without a STOP).
+ if isinstance(error, NotImplementedError) and main is not None:
+ # file is not peekable, but we have main.
+ return None
+ raise UnpicklingError("unable to identify main module") from error
+
+def load_module(
+ filename: Union[str, os.PathLike] = None,
+ module: Optional[Union[ModuleType, str]] = None,
+ **kwds
+) -> Optional[ModuleType]:
+ """Update the selected module (default is :py:mod:`__main__`) with
+ the state saved at ``filename``.
+
+ Restore a module to the state saved with :py:func:`dump_module`. The
+ saved module can be :py:mod:`__main__` (e.g. an interpreter session),
+ an imported module, or a module-type object (e.g. created with
+ :py:class:`~types.ModuleType`).
+
+ When restoring the state of a non-importable module-type object, the
+ current instance of this module may be passed as the argument ``main``.
+ Otherwise, a new instance is created with :py:class:`~types.ModuleType`
+ and returned.
+
+ Args:
+ filename: a path-like object or a readable stream. If `None`
+ (the default), read from a named file in a temporary directory.
+ module: a module object or the name of an importable module;
+ the module name and kind (i.e. imported or non-imported) must
+ match the name and kind of the module stored at ``filename``.
+ **kwds: extra keyword arguments passed to :py:class:`Unpickler()`.
+
+ Raises:
+ :py:exc:`UnpicklingError`: if unpickling fails.
+ :py:exc:`ValueError`: if the argument ``main`` and module saved
+ at ``filename`` are incompatible.
+
+ Returns:
+ A module object, if the saved module is not :py:mod:`__main__` or
+ a module instance wasn't provided with the argument ``main``.
+
+ Examples:
+
+ - Save the state of some modules:
+
+ >>> import dill
+ >>> squared = lambda x: x*x
+ >>> dill.dump_module() # save state of __main__ to /tmp/session.pkl
+ >>>
+ >>> import pox # an imported module
+ >>> pox.plus_one = lambda x: x+1
+ >>> dill.dump_module('pox_session.pkl', module=pox)
+ >>>
+ >>> from types import ModuleType
+ >>> foo = ModuleType('foo') # a module-type object
+ >>> foo.values = [1,2,3]
+ >>> import math
+ >>> foo.sin = math.sin
+ >>> dill.dump_module('foo_session.pkl', module=foo, refimported=True)
+
+ - Restore the state of the interpreter:
+
+ >>> import dill
+ >>> dill.load_module() # updates __main__ from /tmp/session.pkl
+ >>> squared(2)
+ 4
+
+ - Load the saved state of an importable module:
+
+ >>> import dill
+ >>> pox = dill.load_module('pox_session.pkl')
+ >>> pox.plus_one(1)
+ 2
+ >>> import sys
+ >>> pox in sys.modules.values()
+ True
+
+ - Load the saved state of a non-importable module-type object:
+
+ >>> import dill
+ >>> foo = dill.load_module('foo_session.pkl')
+ >>> [foo.sin(x) for x in foo.values]
+ [0.8414709848078965, 0.9092974268256817, 0.1411200080598672]
+ >>> import math
+ >>> foo.sin is math.sin # foo.sin was saved by reference
+ True
+ >>> import sys
+ >>> foo in sys.modules.values()
+ False
+
+ - Update the state of a non-importable module-type object:
+
+ >>> import dill
+ >>> from types import ModuleType
+ >>> foo = ModuleType('foo')
+ >>> foo.values = ['a','b']
+ >>> foo.sin = lambda x: x*x
+ >>> dill.load_module('foo_session.pkl', module=foo)
+ >>> [foo.sin(x) for x in foo.values]
+ [0.8414709848078965, 0.9092974268256817, 0.1411200080598672]
+
+ *Changed in version 0.3.6:* Function ``load_session()`` was renamed to
+ ``load_module()``. Parameter ``main`` was renamed to ``module``.
+
+ See also:
+ :py:func:`load_module_asdict` to load the contents of module saved
+ with :py:func:`dump_module` into a dictionary.
+ """
+ if 'main' in kwds:
+ warnings.warn(
+ "The argument 'main' has been renamed 'module'.",
+ PendingDeprecationWarning
+ )
+ if module is not None:
+ raise TypeError("both 'module' and 'main' arguments were used")
+ module = kwds.pop('main')
+ main = module
+ if hasattr(filename, 'read'):
+ file = filename
+ else:
+ if filename is None:
+ filename = str(TEMPDIR/'session.pkl')
+ file = open(filename, 'rb')
+ try:
+ file = _make_peekable(file)
+ #FIXME: dill.settings are disabled
+ unpickler = Unpickler(file, **kwds)
+ unpickler._session = True
+
+ # Resolve unpickler._main
+ pickle_main = _identify_module(file, main)
+ if main is None and pickle_main is not None:
+ main = pickle_main
+ if isinstance(main, str):
+ if main.startswith('__runtime__.'):
+ # Create runtime module to load the session into.
+ main = ModuleType(main.partition('.')[-1])
+ else:
+ main = _import_module(main)
+ if main is not None:
+ if not isinstance(main, ModuleType):
+ raise TypeError("%r is not a module" % main)
+ unpickler._main = main
+ else:
+ main = unpickler._main
+
+ # Check against the pickle's main.
+ is_main_imported = _is_imported_module(main)
+ if pickle_main is not None:
+ is_runtime_mod = pickle_main.startswith('__runtime__.')
+ if is_runtime_mod:
+ pickle_main = pickle_main.partition('.')[-1]
+ error_msg = "can't update{} module{} %r with the saved state of{} module{} %r"
+ if is_runtime_mod and is_main_imported:
+ raise ValueError(
+ error_msg.format(" imported", "", "", "-type object")
+ % (main.__name__, pickle_main)
+ )
+ if not is_runtime_mod and not is_main_imported:
+ raise ValueError(
+ error_msg.format("", "-type object", " imported", "")
+ % (pickle_main, main.__name__)
+ )
+ if main.__name__ != pickle_main:
+ raise ValueError(error_msg.format("", "", "", "") % (main.__name__, pickle_main))
+
+ # This is for find_class() to be able to locate it.
+ if not is_main_imported:
+ runtime_main = '__runtime__.%s' % main.__name__
+ sys.modules[runtime_main] = main
+
+ loaded = unpickler.load()
+ finally:
+ if not hasattr(filename, 'read'): # if newly opened file
+ file.close()
+ try:
+ del sys.modules[runtime_main]
+ except (KeyError, NameError):
+ pass
+ assert loaded is main
+ _restore_modules(unpickler, main)
+ if main is _main_module or main is module:
+ return None
+ else:
+ return main
+
+# Backward compatibility.
+def load_session(filename=None, main=None, **kwds):
+ warnings.warn("load_session() has been renamed load_module().", PendingDeprecationWarning)
+ load_module(filename, module=main, **kwds)
+load_session.__doc__ = load_module.__doc__
+
+def load_module_asdict(
+ filename: Union[str, os.PathLike] = None,
+ update: bool = False,
+ **kwds
+) -> dict:
+ """
+ Load the contents of a saved module into a dictionary.
+
+ ``load_module_asdict()`` is the near-equivalent of::
+
+ lambda filename: vars(dill.load_module(filename)).copy()
+
+ however, does not alter the original module. Also, the path of
+ the loaded module is stored in the ``__session__`` attribute.
+
+ Args:
+ filename: a path-like object or a readable stream. If `None`
+ (the default), read from a named file in a temporary directory.
+ update: if `True`, initialize the dictionary with the current state
+ of the module prior to loading the state stored at filename.
+ **kwds: extra keyword arguments passed to :py:class:`Unpickler()`
+
+ Raises:
+ :py:exc:`UnpicklingError`: if unpickling fails
+
+ Returns:
+ A copy of the restored module's dictionary.
+
+ Note:
+ If ``update`` is True, the corresponding module may first be imported
+ into the current namespace before the saved state is loaded from
+ filename to the dictionary. Note that any module that is imported into
+ the current namespace as a side-effect of using ``update`` will not be
+ modified by loading the saved module in filename to a dictionary.
+
+ Example:
+ >>> import dill
+ >>> alist = [1, 2, 3]
+ >>> anum = 42
+ >>> dill.dump_module()
+ >>> anum = 0
+ >>> new_var = 'spam'
+ >>> main = dill.load_module_asdict()
+ >>> main['__name__'], main['__session__']
+ ('__main__', '/tmp/session.pkl')
+ >>> main is globals() # loaded objects don't reference globals
+ False
+ >>> main['alist'] == alist
+ True
+ >>> main['alist'] is alist # was saved by value
+ False
+ >>> main['anum'] == anum # changed after the session was saved
+ False
+ >>> new_var in main # would be True if the option 'update' was set
+ False
+ """
+ if 'module' in kwds:
+ raise TypeError("'module' is an invalid keyword argument for load_module_asdict()")
+ if hasattr(filename, 'read'):
+ file = filename
+ else:
+ if filename is None:
+ filename = str(TEMPDIR/'session.pkl')
+ file = open(filename, 'rb')
+ try:
+ file = _make_peekable(file)
+ main_name = _identify_module(file)
+ old_main = sys.modules.get(main_name)
+ main = ModuleType(main_name)
+ if update:
+ if old_main is None:
+ old_main = _import_module(main_name)
+ main.__dict__.update(old_main.__dict__)
+ else:
+ main.__builtins__ = __builtin__
+ sys.modules[main_name] = main
+ load_module(file, **kwds)
+ finally:
+ if not hasattr(filename, 'read'): # if newly opened file
+ file.close()
+ try:
+ if old_main is None:
+ del sys.modules[main_name]
+ else:
+ sys.modules[main_name] = old_main
+ except NameError: # failed before setting old_main
+ pass
+ main.__session__ = str(filename)
+ return main.__dict__
+
+
+# Internal exports for backward compatibility with dill v0.3.5.1
+# Can't be placed in dill._dill because of circular import problems.
+for name in (
+ '_lookup_module', '_module_map', '_restore_modules', '_stash_modules',
+ 'dump_session', 'load_session' # backward compatibility functions
+):
+ setattr(_dill, name, globals()[name])
+del name
diff --git a/venv/Lib/site-packages/dill/settings.py b/venv/Lib/site-packages/dill/settings.py
new file mode 100644
index 0000000000000000000000000000000000000000..eb07feffc197c60262c049e6adc3fd4c2aa0e822
--- /dev/null
+++ b/venv/Lib/site-packages/dill/settings.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+#
+# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
+# Copyright (c) 2008-2016 California Institute of Technology.
+# Copyright (c) 2016-2026 The Uncertainty Quantification Foundation.
+# License: 3-clause BSD. The full license text is available at:
+# - https://github.com/uqfoundation/dill/blob/master/LICENSE
+"""
+global settings for Pickler
+"""
+
+from pickle import DEFAULT_PROTOCOL
+
+settings = {
+ #'main' : None,
+ 'protocol' : DEFAULT_PROTOCOL,
+ 'byref' : False,
+ #'strictio' : False,
+ 'fmode' : 0, #HANDLE_FMODE
+ 'recurse' : False,
+ 'ignore' : False,
+}
+
+del DEFAULT_PROTOCOL
+
diff --git a/venv/Lib/site-packages/dill/source.py b/venv/Lib/site-packages/dill/source.py
new file mode 100644
index 0000000000000000000000000000000000000000..b77d9fbd7a0ff92dc5d24721b9cba712f10cf067
--- /dev/null
+++ b/venv/Lib/site-packages/dill/source.py
@@ -0,0 +1,1025 @@
+#!/usr/bin/env python
+#
+# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
+# Copyright (c) 2008-2016 California Institute of Technology.
+# Copyright (c) 2016-2026 The Uncertainty Quantification Foundation.
+# License: 3-clause BSD. The full license text is available at:
+# - https://github.com/uqfoundation/dill/blob/master/LICENSE
+#
+# inspired by inspect.py from Python-2.7.6
+# inspect.py author: 'Ka-Ping Yee '
+# inspect.py merged into original dill.source by Mike McKerns 4/13/14
+"""
+Extensions to python's 'inspect' module, which can be used
+to retrieve information from live python objects. The methods
+defined in this module are augmented to facilitate access to
+source code of interactively defined functions and classes,
+as well as provide access to source code for objects defined
+in a file.
+"""
+
+__all__ = ['findsource', 'getsourcelines', 'getsource', 'indent', 'outdent', \
+ '_wrap', 'dumpsource', 'getname', '_namespace', 'getimport', \
+ '_importable', 'importable','isdynamic', 'isfrommain']
+
+import linecache
+import re
+from inspect import (getblock, getfile, getmodule, getsourcefile, indentsize,
+ isbuiltin, isclass, iscode, isframe, isfunction, ismethod,
+ ismodule, istraceback)
+from tokenize import TokenError
+
+from ._dill import IS_IPYTHON
+
+
+def isfrommain(obj):
+ "check if object was built in __main__"
+ module = getmodule(obj)
+ if module and module.__name__ == '__main__':
+ return True
+ return False
+
+
+def isdynamic(obj):
+ "check if object was built in the interpreter"
+ try: file = getfile(obj)
+ except TypeError: file = None
+ if file == '' and isfrommain(obj):
+ return True
+ return False
+
+
+def _matchlambda(func, line):
+ """check if lambda object 'func' matches raw line of code 'line'"""
+ from .detect import code as getcode
+ from .detect import freevars, globalvars, varnames
+ dummy = lambda : '__this_is_a_big_dummy_function__'
+ # process the line (removing leading whitespace, etc)
+ lhs,rhs = line.split('lambda ',1)[-1].split(":", 1) #FIXME: if !1 inputs
+ try: #FIXME: unsafe
+ _ = eval("lambda %s : %s" % (lhs,rhs), globals(),locals())
+ except Exception: _ = dummy
+ # get code objects, for comparison
+ _, code = getcode(_).co_code, getcode(func).co_code
+ # check if func is in closure
+ _f = [line.count(i) for i in freevars(func).keys()]
+ if not _f: # not in closure
+ # check if code matches
+ if _ == code: return True
+ return False
+ # weak check on freevars
+ if not all(_f): return False #XXX: VERY WEAK
+ # weak check on varnames and globalvars
+ _f = varnames(func)
+ _f = [line.count(i) for i in _f[0]+_f[1]]
+ if _f and not all(_f): return False #XXX: VERY WEAK
+ _f = [line.count(i) for i in globalvars(func).keys()]
+ if _f and not all(_f): return False #XXX: VERY WEAK
+ # check if func is a double lambda
+ if (line.count('lambda ') > 1) and (lhs in freevars(func).keys()):
+ _lhs,_rhs = rhs.split('lambda ',1)[-1].split(":",1) #FIXME: if !1 inputs
+ try: #FIXME: unsafe
+ _f = eval("lambda %s : %s" % (_lhs,_rhs), globals(),locals())
+ except Exception: _f = dummy
+ # get code objects, for comparison
+ _, code = getcode(_f).co_code, getcode(func).co_code
+ if len(_) != len(code): return False
+ #NOTE: should be same code same order, but except for 't' and '\x88'
+ _ = set((i,j) for (i,j) in zip(_,code) if i != j)
+ if len(_) != 1: return False #('t','\x88')
+ return True
+ # check indentsize
+ if not indentsize(line): return False #FIXME: is this a good check???
+ # check if code 'pattern' matches
+ #XXX: or pattern match against dis.dis(code)? (or use uncompyle2?)
+ _ = _.split(_[0]) # 't' #XXX: remove matching values if starts the same?
+ _f = code.split(code[0]) # '\x88'
+ #NOTE: should be same code different order, with different first element
+ _ = dict(re.match(r'([\W\D\S])(.*)', _[i]).groups() for i in range(1,len(_)))
+ _f = dict(re.match(r'([\W\D\S])(.*)', _f[i]).groups() for i in range(1,len(_f)))
+ if (_.keys() == _f.keys()) and (sorted(_.values()) == sorted(_f.values())):
+ return True
+ return False
+
+
+def findsource(object):
+ """Return the entire source file and starting line number for an object.
+ For interactively-defined objects, the 'file' is the interpreter's history.
+
+ The argument may be a module, class, method, function, traceback, frame,
+ or code object. The source code is returned as a list of all the lines
+ in the file and the line number indexes a line in that list. An IOError
+ is raised if the source code cannot be retrieved, while a TypeError is
+ raised for objects where the source code is unavailable (e.g. builtins)."""
+
+ module = getmodule(object)
+ try: file = getfile(module)
+ except TypeError: file = None
+ is_module_main = (module and module.__name__ == '__main__' and not file)
+ if IS_IPYTHON and is_module_main:
+ #FIXME: quick fix for functions and classes in IPython interpreter
+ try:
+ file = getfile(object)
+ sourcefile = getsourcefile(object)
+ except TypeError:
+ if isclass(object):
+ for object_method in filter(isfunction, object.__dict__.values()):
+ # look for a method of the class
+ file_candidate = getfile(object_method)
+ if not file_candidate.startswith('': pat1 = r'(.*(?':
+ pat1 = r'(.*(?'
+ if stdin:
+ lnum = len(lines) - 1 # can't get lnum easily, so leverage pat
+ if not pat1: pat1 = r'^(\s*def\s)|(.*(? 0: #XXX: won't find decorators in ?
+ line = lines[lnum]
+ if pat1.match(line):
+ if not stdin: break # co_firstlineno does the job
+ if name == '': # hackery needed to confirm a match
+ if _matchlambda(obj, line): break
+ else: # not a lambda, just look for the name
+ if name in line: # need to check for decorator...
+ hats = 0
+ for _lnum in range(lnum-1,-1,-1):
+ if pat2.match(lines[_lnum]): hats += 1
+ else: break
+ lnum = lnum - hats
+ break
+ lnum = lnum - 1
+ return lines, lnum
+
+ try: # turn instances into classes
+ if not isclass(object) and isclass(type(object)): # __class__
+ object = object.__class__ #XXX: sometimes type(class) is better?
+ #XXX: we don't find how the instance was built
+ except AttributeError: pass
+ if isclass(object):
+ name = object.__name__
+ pat = re.compile(r'^(\s*)class\s*' + name + r'\b')
+ # make some effort to find the best matching class definition:
+ # use the one with the least indentation, which is the one
+ # that's most probably not inside a function definition.
+ candidates = []
+ for i in range(len(lines)-1,-1,-1):
+ match = pat.match(lines[i])
+ if match:
+ # if it's at toplevel, it's already the best one
+ if lines[i][0] == 'c':
+ return lines, i
+ # else add whitespace to candidate list
+ candidates.append((match.group(1), i))
+ if candidates:
+ # this will sort by whitespace, and by line number,
+ # less whitespace first #XXX: should sort high lnum before low
+ candidates.sort()
+ return lines, candidates[0][1]
+ else:
+ raise IOError('could not find class definition')
+ raise IOError('could not find code object')
+
+
+def getblocks(object, lstrip=False, enclosing=False, locate=False):
+ """Return a list of source lines and starting line number for an object.
+ Interactively-defined objects refer to lines in the interpreter's history.
+
+ If enclosing=True, then also return any enclosing code.
+ If lstrip=True, ensure there is no indentation in the first line of code.
+ If locate=True, then also return the line number for the block of code.
+
+ DEPRECATED: use 'getsourcelines' instead
+ """
+ lines, lnum = findsource(object)
+
+ if ismodule(object):
+ if lstrip: lines = _outdent(lines)
+ return ([lines], [0]) if locate is True else [lines]
+
+ #XXX: 'enclosing' means: closures only? or classes and files?
+ indent = indentsize(lines[lnum])
+ block = getblock(lines[lnum:]) #XXX: catch any TokenError here?
+
+ if not enclosing or not indent:
+ if lstrip: block = _outdent(block)
+ return ([block], [lnum]) if locate is True else [block]
+
+ pat1 = r'^(\s*def\s)|(.*(? indent: #XXX: should be >= ?
+ line += len(code) - skip
+ elif target in ''.join(code):
+ blocks.append(code) # save code block as the potential winner
+ _lnum.append(line - skip) # save the line number for the match
+ line += len(code) - skip
+ else:
+ line += 1
+ skip = 0
+ # find skip: the number of consecutive decorators
+ elif pat2.match(lines[line]):
+ try: code = getblock(lines[line:])
+ except TokenError: code = [lines[line]]
+ skip = 1
+ for _line in code[1:]: # skip lines that are decorators
+ if not pat2.match(_line): break
+ skip += 1
+ line += skip
+ # no match: reset skip and go to the next line
+ else:
+ line +=1
+ skip = 0
+
+ if not blocks:
+ blocks = [block]
+ _lnum = [lnum]
+ if lstrip: blocks = [_outdent(block) for block in blocks]
+ # return last match
+ return (blocks, _lnum) if locate is True else blocks
+
+
+def getsourcelines(object, lstrip=False, enclosing=False):
+ """Return a list of source lines and starting line number for an object.
+ Interactively-defined objects refer to lines in the interpreter's history.
+
+ The argument may be a module, class, method, function, traceback, frame,
+ or code object. The source code is returned as a list of the lines
+ corresponding to the object and the line number indicates where in the
+ original source file the first line of code was found. An IOError is
+ raised if the source code cannot be retrieved, while a TypeError is
+ raised for objects where the source code is unavailable (e.g. builtins).
+
+ If lstrip=True, ensure there is no indentation in the first line of code.
+ If enclosing=True, then also return any enclosing code."""
+ code, n = getblocks(object, lstrip=lstrip, enclosing=enclosing, locate=True)
+ return code[-1], n[-1]
+
+
+#NOTE: broke backward compatibility 4/16/14 (was lstrip=True, force=True)
+def getsource(object, alias='', lstrip=False, enclosing=False, \
+ force=False, builtin=False):
+ """Return the text of the source code for an object. The source code for
+ interactively-defined objects are extracted from the interpreter's history.
+
+ The argument may be a module, class, method, function, traceback, frame,
+ or code object. The source code is returned as a single string. An
+ IOError is raised if the source code cannot be retrieved, while a
+ TypeError is raised for objects where the source code is unavailable
+ (e.g. builtins).
+
+ If alias is provided, then add a line of code that renames the object.
+ If lstrip=True, ensure there is no indentation in the first line of code.
+ If enclosing=True, then also return any enclosing code.
+ If force=True, catch (TypeError,IOError) and try to use import hooks.
+ If builtin=True, force an import for any builtins
+ """
+ # hascode denotes a callable
+ hascode = _hascode(object)
+ # is a class instance type (and not in builtins)
+ instance = _isinstance(object)
+
+ # get source lines; if fail, try to 'force' an import
+ try: # fails for builtins, and other assorted object types
+ lines, lnum = getsourcelines(object, enclosing=enclosing)
+ except (TypeError, IOError): # failed to get source, resort to import hooks
+ if not force: # don't try to get types that findsource can't get
+ raise
+ if not getmodule(object): # get things like 'None' and '1'
+ if not instance: return getimport(object, alias, builtin=builtin)
+ # special handling (numpy arrays, ...)
+ _import = getimport(object, builtin=builtin)
+ name = getname(object, force=True)
+ _alias = "%s = " % alias if alias else ""
+ if alias == name: _alias = ""
+ return _import+_alias+"%s\n" % name
+ else: #FIXME: could use a good bit of cleanup, since using getimport...
+ if not instance: return getimport(object, alias, builtin=builtin)
+ # now we are dealing with an instance...
+ name = object.__class__.__name__
+ module = object.__module__
+ if not name.isidentifier() or not all(i.isidentifier() for i in module.split('.')): #XXX: does exclusing malicious code exclude valid use?
+ raise SyntaxError('invalid syntax')
+ if module in ['builtins','__builtin__']:
+ return getimport(object, alias, builtin=builtin)
+ else: #FIXME: leverage getimport? use 'from module import name'?
+ lines, lnum = ["%s = __import__('%s', fromlist=['%s']).%s\n" % (name,module,name,name)], 0
+ obj = eval(lines[0].lstrip(name + ' = '))
+ lines, lnum = getsourcelines(obj, enclosing=enclosing)
+
+ # strip leading indent (helps ensure can be imported)
+ if lstrip or alias:
+ lines = _outdent(lines)
+
+ # instantiate, if there's a nice repr #XXX: BAD IDEA???
+ if instance: #and force: #XXX: move into findsource or getsourcelines ?
+ if '(' in repr(object): lines.append('%r\n' % object)
+ #else: #XXX: better to somehow to leverage __reduce__ ?
+ # reconstructor,args = object.__reduce__()
+ # _ = reconstructor(*args)
+ else: # fall back to serialization #XXX: bad idea?
+ #XXX: better not duplicate work? #XXX: better new/enclose=True?
+ lines = dumpsource(object, alias='', new=force, enclose=False)
+ lines, lnum = [line+'\n' for line in lines.split('\n')][:-1], 0
+ #else: object.__code__ # raise AttributeError
+
+ # add an alias to the source code
+ if alias:
+ if hascode:
+ skip = 0
+ for line in lines: # skip lines that are decorators
+ if not line.startswith('@'): break
+ skip += 1
+ #XXX: use regex from findsource / getsourcelines ?
+ if lines[skip].lstrip().startswith('def '): # we have a function
+ if alias != object.__name__:
+ lines.append('\n%s = %s\n' % (alias, object.__name__))
+ elif 'lambda ' in lines[skip]: # we have a lambda
+ if alias != lines[skip].split('=')[0].strip():
+ lines[skip] = '%s = %s' % (alias, lines[skip])
+ else: # ...try to use the object's name
+ if alias != object.__name__:
+ lines.append('\n%s = %s\n' % (alias, object.__name__))
+ else: # class or class instance
+ if instance:
+ if alias != lines[-1].split('=')[0].strip():
+ lines[-1] = ('%s = ' % alias) + lines[-1]
+ else:
+ name = getname(object, force=True) or object.__name__
+ if alias != name:
+ lines.append('\n%s = %s\n' % (alias, name))
+ return ''.join(lines)
+
+
+def _hascode(object):
+ '''True if object has an attribute that stores it's __code__'''
+ return getattr(object,'__code__',None) or getattr(object,'func_code',None)
+
+def _isinstance(object):
+ '''True if object is a class instance type (and is not a builtin)'''
+ if _hascode(object) or isclass(object) or ismodule(object):
+ return False
+ if istraceback(object) or isframe(object) or iscode(object):
+ return False
+ # special handling (numpy arrays, ...)
+ if not getmodule(object) and getmodule(type(object)).__name__ in ['numpy']:
+ return True
+# # check if is instance of a builtin
+# if not getmodule(object) and getmodule(type(object)).__name__ in ['__builtin__','builtins']:
+# return False
+ _types = ('")
+ if not repr(type(object)).startswith(_types): #FIXME: weak hack
+ return False
+ if not getmodule(object) or object.__module__ in ['builtins','__builtin__'] or getname(object, force=True) in ['array']:
+ return False
+ return True # by process of elimination... it's what we want
+
+
+def _intypes(object):
+ '''check if object is in the 'types' module'''
+ import types
+ # allow user to pass in object or object.__name__
+ if type(object) is not type(''):
+ object = getname(object, force=True)
+ if object == 'ellipsis': object = 'EllipsisType'
+ return True if hasattr(types, object) else False
+
+
+def _isstring(object): #XXX: isstringlike better?
+ '''check if object is a string-like type'''
+ return isinstance(object, (str, bytes))
+
+
+def indent(code, spaces=4):
+ '''indent a block of code with whitespace (default is 4 spaces)'''
+ indent = indentsize(code)
+ from numbers import Integral
+ if isinstance(spaces, Integral): spaces = ' '*spaces
+ # if '\t' is provided, will indent with a tab
+ nspaces = indentsize(spaces)
+ # blank lines (etc) need to be ignored
+ lines = code.split('\n')
+## stq = "'''"; dtq = '"""'
+## in_stq = in_dtq = False
+ for i in range(len(lines)):
+ #FIXME: works... but shouldn't indent 2nd+ lines of multiline doc
+ _indent = indentsize(lines[i])
+ if indent > _indent: continue
+ lines[i] = spaces+lines[i]
+## #FIXME: may fail when stq and dtq in same line (depends on ordering)
+## nstq, ndtq = lines[i].count(stq), lines[i].count(dtq)
+## if not in_dtq and not in_stq:
+## lines[i] = spaces+lines[i] # we indent
+## # entering a comment block
+## if nstq%2: in_stq = not in_stq
+## if ndtq%2: in_dtq = not in_dtq
+## # leaving a comment block
+## elif in_dtq and ndtq%2: in_dtq = not in_dtq
+## elif in_stq and nstq%2: in_stq = not in_stq
+## else: pass
+ if lines[-1].strip() == '': lines[-1] = ''
+ return '\n'.join(lines)
+
+
+def _outdent(lines, spaces=None, all=True):
+ '''outdent lines of code, accounting for docs and line continuations'''
+ indent = indentsize(lines[0])
+ if spaces is None or spaces > indent or spaces < 0: spaces = indent
+ for i in range(len(lines) if all else 1):
+ #FIXME: works... but shouldn't outdent 2nd+ lines of multiline doc
+ _indent = indentsize(lines[i])
+ if spaces > _indent: _spaces = _indent
+ else: _spaces = spaces
+ lines[i] = lines[i][_spaces:]
+ return lines
+
+def outdent(code, spaces=None, all=True):
+ '''outdent a block of code (default is to strip all leading whitespace)'''
+ indent = indentsize(code)
+ if spaces is None or spaces > indent or spaces < 0: spaces = indent
+ #XXX: will this delete '\n' in some cases?
+ if not all: return code[spaces:]
+ return '\n'.join(_outdent(code.split('\n'), spaces=spaces, all=all))
+
+
+# _wrap provides an wrapper to correctly exec and load into locals
+__globals__ = globals()
+__locals__ = locals()
+def _wrap(f):
+ """ encapsulate a function and it's __import__ """
+ def func(*args, **kwds):
+ try:
+ # _ = eval(getsource(f, force=True)) #XXX: safer but less robust
+ exec(getimportable(f, alias='_'), __globals__, __locals__)
+ except Exception:
+ raise ImportError('cannot import name ' + f.__name__)
+ return _(*args, **kwds)
+ func.__name__ = f.__name__
+ func.__doc__ = f.__doc__
+ return func
+
+
+def _enclose(object, alias=''): #FIXME: needs alias to hold returned object
+ """create a function enclosure around the source of some object"""
+ #XXX: dummy and stub should append a random string
+ dummy = '__this_is_a_big_dummy_enclosing_function__'
+ stub = '__this_is_a_stub_variable__'
+ code = 'def %s():\n' % dummy
+ code += indent(getsource(object, alias=stub, lstrip=True, force=True))
+ code += indent('return %s\n' % stub)
+ if alias: code += '%s = ' % alias
+ code += '%s(); del %s\n' % (dummy, dummy)
+ #code += "globals().pop('%s',lambda :None)()\n" % dummy
+ return code
+
+
+def dumpsource(object, alias='', new=False, enclose=True):
+ """'dump to source', where the code includes a pickled object.
+
+ If new=True and object is a class instance, then create a new
+ instance using the unpacked class source code. If enclose, then
+ create the object inside a function enclosure (thus minimizing
+ any global namespace pollution).
+ """
+ from dill import dumps
+ pik = repr(dumps(object))
+ code = 'import dill\n'
+ if enclose:
+ stub = '__this_is_a_stub_variable__' #XXX: *must* be same _enclose.stub
+ pre = '%s = ' % stub
+ new = False #FIXME: new=True doesn't work with enclose=True
+ else:
+ stub = alias
+ pre = '%s = ' % stub if alias else alias
+
+ # if a 'new' instance is not needed, then just dump and load
+ if not new or not _isinstance(object):
+ code += pre + 'dill.loads(%s)\n' % pik
+ else: #XXX: other cases where source code is needed???
+ code += getsource(object.__class__, alias='', lstrip=True, force=True)
+ mod = repr(object.__module__) # should have a module (no builtins here)
+ code += pre + 'dill.loads(%s.replace(b%s,bytes(__name__,"UTF-8")))\n' % (pik,mod)
+ #code += 'del %s' % object.__class__.__name__ #NOTE: kills any existing!
+
+ if enclose:
+ # generation of the 'enclosure'
+ dummy = '__this_is_a_big_dummy_object__'
+ dummy = _enclose(dummy, alias=alias)
+ # hack to replace the 'dummy' with the 'real' code
+ dummy = dummy.split('\n')
+ code = dummy[0]+'\n' + indent(code) + '\n'.join(dummy[-3:])
+
+ return code #XXX: better 'dumpsourcelines', returning list of lines?
+
+
+def getname(obj, force=False, fqn=False): #XXX: throw(?) to raise error on fail?
+ """get the name of the object. for lambdas, get the name of the pointer """
+ if fqn: return '.'.join(_namespace(obj)) #NOTE: returns 'type'
+ module = getmodule(obj)
+ if not module: # things like "None" and "1"
+ if not force: return None #NOTE: returns 'instance' NOT 'type' #FIXME?
+ # handle some special cases
+ if hasattr(obj, 'dtype') and not obj.shape:
+ return getname(obj.__class__) + "(" + repr(obj.tolist()) + ")"
+ return repr(obj)
+ try:
+ #XXX: 'wrong' for decorators and curried functions ?
+ # if obj.func_closure: ...use logic from getimportable, etc ?
+ name = obj.__name__
+ if name == '':
+ return getsource(obj).split('=',1)[0].strip()
+ # handle some special cases
+ if module.__name__ in ['builtins','__builtin__']:
+ if name == 'ellipsis': name = 'EllipsisType'
+ return name
+ except AttributeError: #XXX: better to just throw AttributeError ?
+ if not force: return None
+ name = repr(obj)
+ if name.startswith('<'): # or name.split('('):
+ return None
+ return name
+
+
+def _namespace(obj):
+ """_namespace(obj); return namespace hierarchy (as a list of names)
+ for the given object. For an instance, find the class hierarchy.
+
+ For example:
+
+ >>> from functools import partial
+ >>> p = partial(int, base=2)
+ >>> _namespace(p)
+ [\'functools\', \'partial\']
+ """
+ # mostly for functions and modules and such
+ #FIXME: 'wrong' for decorators and curried functions
+ try: #XXX: needs some work and testing on different types
+ module = qual = str(getmodule(obj)).split()[1].strip('>').strip('"').strip("'")
+ qual = qual.split('.')
+ if ismodule(obj):
+ return qual
+ # get name of a lambda, function, etc
+ name = getname(obj) or obj.__name__ # failing, raise AttributeError
+ # check special cases (NoneType, ...)
+ if module in ['builtins','__builtin__']: # BuiltinFunctionType
+ if _intypes(name): return ['types'] + [name]
+ return qual + [name] #XXX: can be wrong for some aliased objects
+ except Exception: pass
+ # special case: numpy.inf and numpy.nan (we don't want them as floats)
+ if str(obj) in ['inf','nan','Inf','NaN']: # is more, but are they needed?
+ return ['numpy'] + [str(obj)]
+ # mostly for classes and class instances and such
+ module = getattr(obj.__class__, '__module__', None)
+ qual = str(obj.__class__)
+ try: qual = qual[qual.index("'")+1:-2]
+ except ValueError: pass # str(obj.__class__) made the 'try' unnecessary
+ qual = qual.split(".")
+ if module in ['builtins','__builtin__']:
+ # check special cases (NoneType, Ellipsis, ...)
+ if qual[-1] == 'ellipsis': qual[-1] = 'EllipsisType'
+ if _intypes(qual[-1]): module = 'types' #XXX: BuiltinFunctionType
+ qual = [module] + qual
+ return qual
+
+
+#NOTE: 05/25/14 broke backward compatibility: added 'alias' as 3rd argument
+def _getimport(head, tail, alias='', verify=True, builtin=False):
+ """helper to build a likely import string from head and tail of namespace.
+ ('head','tail') are used in the following context: "from head import tail"
+
+ If verify=True, then test the import string before returning it.
+ If builtin=True, then force an import for builtins where possible.
+ If alias is provided, then rename the object on import.
+ """
+ # special handling for a few common types
+ if tail in ['Ellipsis', 'NotImplemented'] and head in ['types']:
+ head = len.__module__
+ elif tail in ['None'] and head in ['types']:
+ _alias = '%s = ' % alias if alias else ''
+ if alias == tail: _alias = ''
+ return _alias+'%s\n' % tail
+ # we don't need to import from builtins, so return ''
+# elif tail in ['NoneType','int','float','long','complex']: return '' #XXX: ?
+ if head in ['builtins','__builtin__']:
+ # special cases (NoneType, Ellipsis, ...) #XXX: BuiltinFunctionType
+ if tail == 'ellipsis': tail = 'EllipsisType'
+ if _intypes(tail): head = 'types'
+ elif not builtin:
+ _alias = '%s = ' % alias if alias else ''
+ if alias == tail: _alias = ''
+ return _alias+'%s\n' % tail
+ else: pass # handle builtins below
+ # get likely import string
+ if not head: _str = "import %s" % tail
+ else: _str = "from %s import %s" % (head, tail)
+ _alias = " as %s\n" % alias if alias else "\n"
+ if alias == tail: _alias = "\n"
+ _str += _alias
+ # FIXME: fails on most decorators, currying, and such...
+ # (could look for magic __wrapped__ or __func__ attr)
+ # (could fix in 'namespace' to check obj for closure)
+ if verify and not head.startswith('dill.'):# weird behavior for dill
+ #print(_str)
+ try: exec(_str) #XXX: check if == obj? (name collision)
+ except ImportError: #XXX: better top-down or bottom-up recursion?
+ _head = head.rsplit(".",1)[0] #(or get all, then compare == obj?)
+ if not _head: raise
+ if _head != head:
+ _str = _getimport(_head, tail, alias, verify)
+ return _str
+
+
+#XXX: rename builtin to force? vice versa? verify to force? (as in getsource)
+#NOTE: 05/25/14 broke backward compatibility: added 'alias' as 2nd argument
+def getimport(obj, alias='', verify=True, builtin=False, enclosing=False):
+ """get the likely import string for the given object
+
+ obj is the object to inspect
+ If verify=True, then test the import string before returning it.
+ If builtin=True, then force an import for builtins where possible.
+ If enclosing=True, get the import for the outermost enclosing callable.
+ If alias is provided, then rename the object on import.
+ """
+ if enclosing:
+ from .detect import outermost
+ _obj = outermost(obj)
+ obj = _obj if _obj else obj
+ # get the namespace
+ qual = _namespace(obj)
+ head = '.'.join(qual[:-1])
+ tail = qual[-1]
+ # for named things... with a nice repr #XXX: move into _namespace?
+ try: # look for '<...>' and be mindful it might be in lists, dicts, etc...
+ name = repr(obj).split('<',1)[1].split('>',1)[1]
+ name = None # we have a 'object'-style repr
+ except Exception: # it's probably something 'importable'
+ if head in ['builtins','__builtin__']:
+ name = repr(obj) #XXX: catch [1,2], (1,2), set([1,2])... others?
+ elif _isinstance(obj):
+ name = getname(obj, force=True).split('(')[0]
+ else:
+ name = repr(obj).split('(')[0]
+ #if not repr(obj).startswith('<'): name = repr(obj).split('(')[0]
+ #else: name = None
+ if name: # try using name instead of tail
+ try: return _getimport(head, name, alias, verify, builtin)
+ except ImportError: pass
+ except SyntaxError:
+ if head in ['builtins','__builtin__']:
+ _alias = '%s = ' % alias if alias else ''
+ if alias == name: _alias = ''
+ return _alias+'%s\n' % name
+ else: pass
+ try:
+ #if type(obj) is type(abs): _builtin = builtin # BuiltinFunctionType
+ #else: _builtin = False
+ return _getimport(head, tail, alias, verify, builtin)
+ except ImportError:
+ raise # could do some checking against obj
+ except SyntaxError:
+ if head in ['builtins','__builtin__']:
+ _alias = '%s = ' % alias if alias else ''
+ if alias == tail: _alias = ''
+ return _alias+'%s\n' % tail
+ raise # could do some checking against obj
+
+
+def _importable(obj, alias='', source=None, enclosing=False, force=True, \
+ builtin=True, lstrip=True):
+ """get an import string (or the source code) for the given object
+
+ This function will attempt to discover the name of the object, or the repr
+ of the object, or the source code for the object. To attempt to force
+ discovery of the source code, use source=True, to attempt to force the
+ use of an import, use source=False; otherwise an import will be sought
+ for objects not defined in __main__. The intent is to build a string
+ that can be imported from a python file. obj is the object to inspect.
+ If alias is provided, then rename the object with the given alias.
+
+ If source=True, use these options:
+ If enclosing=True, then also return any enclosing code.
+ If force=True, catch (TypeError,IOError) and try to use import hooks.
+ If lstrip=True, ensure there is no indentation in the first line of code.
+
+ If source=False, use these options:
+ If enclosing=True, get the import for the outermost enclosing callable.
+ If force=True, then don't test the import string before returning it.
+ If builtin=True, then force an import for builtins where possible.
+ """
+ if source is None:
+ source = True if isfrommain(obj) else False
+ if source: # first try to get the source
+ try:
+ return getsource(obj, alias, enclosing=enclosing, \
+ force=force, lstrip=lstrip, builtin=builtin)
+ except Exception: pass
+ try:
+ if not _isinstance(obj):
+ return getimport(obj, alias, enclosing=enclosing, \
+ verify=(not force), builtin=builtin)
+ # first 'get the import', then 'get the instance'
+ _import = getimport(obj, enclosing=enclosing, \
+ verify=(not force), builtin=builtin)
+ name = getname(obj, force=True)
+ if not name:
+ raise AttributeError("object has no atribute '__name__'")
+ _alias = "%s = " % alias if alias else ""
+ if alias == name: _alias = ""
+ return _import+_alias+"%s\n" % name
+
+ except Exception: pass
+ if not source: # try getsource, only if it hasn't been tried yet
+ try:
+ return getsource(obj, alias, enclosing=enclosing, \
+ force=force, lstrip=lstrip, builtin=builtin)
+ except Exception: pass
+ # get the name (of functions, lambdas, and classes)
+ # or hope that obj can be built from the __repr__
+ #XXX: what to do about class instances and such?
+ obj = getname(obj, force=force)
+ # we either have __repr__ or __name__ (or None)
+ if not obj or obj.startswith('<'):
+ raise AttributeError("object has no atribute '__name__'")
+ _alias = '%s = ' % alias if alias else ''
+ if alias == obj: _alias = ''
+ return _alias+'%s\n' % obj
+ #XXX: possible failsafe... (for example, for instances when source=False)
+ # "import dill; result = dill.loads(); # repr(
+ 
+ 
+ 
+ 
+ 
+ 
+
+